diff --git a/README-CN.md b/README-CN.md index 437d671bb97a1009f5d48d2423a3d17f75bf1494..a5d239a53296b5a7d96a3e1624879386ab57bf13 100644 --- a/README-CN.md +++ b/README-CN.md @@ -60,7 +60,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev 为了在 Ubuntu/Debian 系统上编译 [taos-tools](https://github.com/taosdata/taos-tools) 需要安装如下软件： ```bash -sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config ``` ### CentOS 7.9 @@ -85,7 +85,7 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel ``` -sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` #### CentOS 8/Rocky Linux @@ -94,7 +94,7 @@ sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgco sudo yum install -y epel-release sudo yum install -y dnf-plugins-core sudo yum config-manager --set-enabled powertools -sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` 注意：由于 snappy 缺乏 pkg-config 支持（参考 [链接](https://github.com/google/snappy/pull/86)），会导致 cmake 提示无法发现 libsnappy，实际上工作正常。 diff --git a/README.md b/README.md index 6aec756ec76d6801bfe72e3c66bc36a325d8245e..05c1c075f0c3b3894c93d0aa2e126a9dc6340758 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sudo apt-get install -y gcc cmake build-essential git libssl-dev To build the [taosTools](https://github.com/taosdata/taos-tools) on Ubuntu/Debian, the following packages need to be installed. ```bash -sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config +sudo apt install build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config ``` ### CentOS 7.9 @@ -85,7 +85,7 @@ sudo dnf install -y gcc gcc-c++ make cmake epel-release git openssl-devel #### CentOS 7.9 ``` -sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` #### CentOS 8/Rocky Linux @@ -94,7 +94,7 @@ sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgco sudo yum install -y epel-release sudo yum install -y dnf-plugins-core sudo yum config-manager --set-enabled powertools -sudo yum install -y zlib-devel xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libstdc++-static openssl-devel +sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson-devel pkgconfig libatomic libatomic-static libstdc++-static openssl-devel ``` Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it leads a cmake prompt libsnappy not found. But snappy still works well. diff --git a/cmake/cmake.define b/cmake/cmake.define index e34785cba6846d133d3d80c334089542cee8ac9d..7410c92525db02c81f14cfcf829eab74ef5c1f91 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -123,21 +123,37 @@ ELSE () SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k") ENDIF () - IF (TD_INTEL_64 OR TD_INTEL_32) - ADD_DEFINITIONS("-msse4.2") - IF("${FMA_SUPPORT}" MATCHES "true") - MESSAGE(STATUS "fma function supported") - ADD_DEFINITIONS("-mfma") - ELSE () - MESSAGE(STATUS "fma function NOT supported") + INCLUDE(CheckCCompilerFlag) + IF (("${CMAKE_C_COMPILER_ID}" MATCHES "Clang") OR ("${CMAKE_C_COMPILER_ID}" MATCHES "AppleClang")) + SET(COMPILER_SUPPORT_SSE42 true) + MESSAGE(STATUS "Always enable sse4.2 for Clang/AppleClang") + ELSE() + CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE42) + ENDIF() + + CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA) + CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX) + CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2) + + IF (COMPILER_SUPPORT_SSE42) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse4.2") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2") + ENDIF() + IF (COMPILER_SUPPORT_FMA) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfma") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfma") + ENDIF() + + IF ("${SIMD_SUPPORT}" MATCHES "true") + IF (COMPILER_SUPPORT_AVX) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") ENDIF() - - IF("${SIMD_SUPPORT}" MATCHES "true") - ADD_DEFINITIONS("-mavx -mavx2") - MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") - ELSE() - MESSAGE(STATUS "SIMD instruction (AVX/AVX2)is NOT ACTIVATED") + IF (COMPILER_SUPPORT_AVX2) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") ENDIF() - ENDIF () + MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED") + ENDIF() ENDIF () diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 711d74fa4cd94cb5c3e7527d816dd876f5f72148..a4bfcaf609cc98f43094dabd993b8e310b9b220f 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -147,7 +147,7 @@ ELSE () ENDIF () ENDIF () -MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR}) +MESSAGE(STATUS "Platform arch:" ${PLATFORM_ARCH_STR}) MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})") MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})") diff --git a/cmake/cmake.version b/cmake/cmake.version index 9faa7c75dd24f5b07d9894d2c3033803bf22f899..b4084bd095b37d4b26024f8fee410a9f42bb0ef3 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.1.8") + SET(TD_VER_NUMBER "3.0.2.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index cc46ef99386420196749b2d8b5886e1d65fa9943..75679a8ff5b229e649cdccb3a992dc0fc00fa8e2 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ff7de07 + GIT_TAG 566540d SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index db727907b79b33f82e8bb3a3369d32431c412a1e..c5fe95da586bdf0927329d1ec121394ead981fdb 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 823fae5 + GIT_TAG b20c9d1 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index 7a3ac6bad3b203a9859e235e92d0865f7147d24c..214580c179b97f46f94096411c57bf0004dc68e3 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,6 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http :::note - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. -- Only data in array format is accepted and so an array must be used even if there is only one row. - The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index a12406fe4353d437c5df9755f8f8e68b0f24282f..0aec22fbc072409916cfa7a7a8ff0d1702fb226b 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -27,7 +27,6 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,9 +60,6 @@ database_option: { - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster. - RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods. -- STRICT: specifies whether strong data consistency is enabled. The default value is off. - - on: Strong consistency is enabled and implemented through the Raft consensus algorithm. In this mode, an operation is considered successful once it is confirmed by half of the nodes in the cluster. - - off: Strong consistency is disabled. In this mode, an operation is considered successful when it is initiated by the local node. - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1. - 1: WAL is enabled but fsync is disabled. - 2: WAL and fsync are both enabled. diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 6145d8c00c97a411699f7decf731399c238ff84e..f0daf4b82a2fdc9d519d151451a1bbd10fd93e31 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -532,7 +532,12 @@ TIMEDIFF(expr1, expr2 [, time_unit]) #### TIMETRUNCATE ```sql -TIMETRUNCATE(expr, time_unit) +TIMETRUNCATE(expr, time_unit [, ignore_timezone]) + +ignore_timezone: { + 0 + | 1 +} ``` **Description**: Truncate the input timestamp with unit specified by `time_unit` @@ -548,7 +553,10 @@ TIMETRUNCATE(expr, time_unit) 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) - The precision of the returned timestamp is same as the precision set for the current data base in use - If the input data is not formatted as a timestamp, the returned value is null. - +- If `1d` is used as `time_unit` to truncate the timestamp, `ignore_timezone` option can be set to indicate if the returned result is affected by client timezone or not. + For example, if client timezone is set to UTC+0800, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) will return '2020-01-01 08:00:00'. + Otherwise, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) will return '2020-01-01 00:00:00'. + If `ignore_timezone` option is omitted, the default value is set to 1. #### TIMEZONE diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 17e4e4d1b0da6d0461c9ab478a9430855379fb12..c47d2da0ebf8b8d3ecc2e51fad659ceb423ad46f 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -10,7 +10,7 @@ Because stream processing is built in to TDengine, you are no longer reliant on ## Create a Stream ```sql -CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery +CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery stream_options: { TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] WATERMARK time @@ -30,6 +30,8 @@ subquery: SELECT [DISTINCT] select_list Session windows, state windows, and sliding windows are supported. When you configure a session or state window for a supertable, you must use PARTITION BY TBNAME. +Subtable Clause defines the naming rules of auto-created subtable, you can see more details in below part: Partitions of Stream. + ```sql window_clause: { SESSION(ts_col, tol_val) @@ -47,6 +49,47 @@ CREATE STREAM avg_vol_s INTO avg_vol AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); ``` +## Partitions of Stream + +A Stream can process data in multiple partitions. Partition rules can be defined by PARTITION BY clause in stream processing. Each partition will have different timelines and windows, and will be processed separately and be written into different subtables of target supertable. + +If a stream is created without PARTITION BY clause, all data will be written into one subtable. + +If a stream is created with PARTITION BY clause without SUBTABLE clause, each partition will be given a random name. + +If a stream is created with PARTITION BY clause and SUBTABLE clause, the name of each partition will be calculated according to SUBTABLE clause. For example: + +```sql +CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); +``` + +IN PARTITION clause, 'tbname', representing each subtable name of source supertable, is given alias 'tname'. And 'tname' is used in SUBTABLE clause. In SUBTABLE clause, each auto created subtable will concat 'new-' and source subtable name as their name. Other expressions are also allowed in SUBTABLE clause, but the output type must be varchar. + +If the output length exceeds the limitation of TDengine(192), the name will be truncated. If the generated name is occupied by some other table, the creation and writing of the new subtable will be failed. + +## Filling history data + +Normally a stream does not process data already or being written into source table when it's being creating. But adding FILL_HISTORY 1 as a stream option when creating the stream will allow it to process data written before and while creating the stream. For example: + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s) +``` + +Combining fill_history option and where clause, stream can processing data of specific time range. For example, only process data after a past time. (In this case, 2020-01-30) + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s) +``` + +As another example, only processing data starting from some past time, and ending at some future time. + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' and ts < '2023-01-01' interval(10s) +``` + +If some streams are totally outdated, and you do not want it to monitor or process anymore, those streams can be manually dropped and output data will be still kept. + + ## Delete a Stream ```sql diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 7b3440ebac2f630a2f6f39c3b270a2196d1d7739..c37738b3f8ddba9879159e04573798dd61efa27e 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -878,8 +878,10 @@ The source code of the sample application is under `TDengine/examples/JDBC`: | taos-jdbcdriver version | major changes | | :---------------------: | :--------------------------------------------: | +| 3.0.3 | fix timestamp resolution error for REST connection in jdk17+ version | | 3.0.1 - 3.0.2 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use 3.0.2 in the JDK 8 environment | | 3.0.0 | Support for TDengine 3.0 | +| 2.0.42 | fix wasNull interface return value in WebSocket connection | | 2.0.41 | fix decode method of username and password in REST connection | | 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | | 2.0.38 | JDBC REST connections add bulk pull function | diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 54031db618b45e3fb314b973f9837b4ce9b4062f..ba8dbb85d4982c5d6c89f5dbe6157bd88a8c00a4 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -59,10 +59,10 @@ The different database framework specifications for various programming language | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | | **Connection Management** | Support | Support | Support | Support | Support | Support | | **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Not supported | Not supported | Not supported | Support | Not supported | Support | -| **Subscription (TMQ) ** | Not supported | Not supported | Not supported | Not supported | Not supported | Support | +| **Parameter Binding** | Not supported | Not supported | support | Support | Not supported | Support | +| **Subscription (TMQ) ** | Not supported | Not supported | support | Not supported | Not supported | Support | | **Schemaless** | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | -| **Bulk Pulling (based on WebSocket) ** | Support | Support | Not Supported | support | Not Supported | Supported | +| **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | support | Support | Support | | **DataFrame** | Not supported | Support | Not supported | Not supported | Not supported | Not supported | :::warning diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index ad00584360acc18dca46998948fd96790965e5b1..870cebb1033f0f0cddaec19b616228c2ac46d87c 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -59,6 +59,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +101,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +112,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +134,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -195,6 +199,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the frist data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 6e08671e344100aa3dcfeba5a10c0d0e304a1a91..19feeb674060cbe0e7ec13ed4e47bb3fd85836cc 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -204,6 +204,12 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-a/--replica ** : Specify the number of replicas when creating the database. The default value is 1. +- **-k/--keep-trying ** : + Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- **-z/--trying-interval ** : + Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + - **-V/--version** : Show version information only. Users should not use it with other parameters. @@ -231,6 +237,10 @@ The parameters listed in this section apply to all function modes. `filetype` must be set to `insert` in the insertion scenario. See [General Configuration Parameters](#General Configuration Parameters) +- ** keep_trying ** : Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- ** trying_interval ** : Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + #### Database related configuration parameters The parameters related to database creation are configured in `dbinfo` in the json configuration file, as follows. The other parameters correspond to the database parameters specified when `create database` in [../../taos-sql/database]. diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index e73441a96b087062b2e3912ed73010fc3e761bb9..9c63b4dc035b6a6a18f8e17f19782527eef88bca 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -19,7 +19,7 @@ Users should not use taosdump to back up raw data, environment settings, hardwar There are two ways to install taosdump: -- Install the taosTools official installer. Please find taosTools from [All download links](https://www.tdengine.com/all-downloads) page and download and install it. +- Install the taosTools official installer. Please find taosTools from [Release History](https://docs.taosdata.com/releases/tools/) page and download and install it. - Compile taos-tools separately and install it. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index bb5516ae700ad7cd5b47a87ed56338c27aa1eab0..b6bfa4bc7d57a9139992a0f1aab528b267e5bd03 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -153,11 +153,11 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Execution policy for query statements | | Unit | None | | Default | 1 | -| Notes | 1: Run queries on vnodes and not on qnodes | +| Value Range | 1: Run queries on vnodes and not on qnodes 2: Run subtasks without scan operators on qnodes and subtasks with scan operators on vnodes. -3: Only run scan operators on vnodes; run all other operators on qnodes. +3: Only run scan operators on vnodes; run all other operators on qnodes. | ### querySmaOptimize @@ -173,6 +173,14 @@ The parameters described in this document by the effect that they have on the sy 1: Enable SMA indexing and perform queries from suitable statements on precomputation results.| +### countAlwaysReturnValue + +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server only | +| Meaning | count()/hyperloglog() return value or not if the result data is NULL | +| Vlue Range | 0：Return empty line，1：Return 0 | +| Default | 1 | ### maxNumOfDistinctRes @@ -307,6 +315,14 @@ The charset that takes effect is UTF-8. | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +### tempDir + +| Attribute | Description | +| -------- | ------------------------------------------ | +| Applicable | Server only | +| Meaning | The directory where to put all the temporary files generated during system running | +| Default | /tmp | + ### minimalTmpDirGB | Attribute | Description | @@ -336,89 +352,6 @@ The charset that takes effect is UTF-8. | Value Range | 0-4096 | | Default Value | 2x the CPU cores | -## Time Parameters - -### statusInterval - -| Attribute | Description | -| -------- | --------------------------- | -| Applicable | Server Only | -| Meaning | the interval of dnode reporting status to mnode | -| Unit | second | -| Value Range | 1-10 | -| Default Value | 1 | - -### shellActivityTimer - -| Attribute | Description | -| -------- | --------------------------------- | -| Applicable | Server and Client | -| Meaning | The interval for TDengine CLI to send heartbeat to mnode | -| Unit | second | -| Value Range | 1-120 | -| Default Value | 3 | - -## Performance Optimization Parameters - -### numOfCommitThreads - -| Attribute | Description | -| -------- | ---------------------- | -| Applicable | Server Only | -| Meaning | Maximum of threads for committing to disk | -| Default Value | | - -## Compression Parameters - -### compressMsgSize - -| Attribute | Description | -| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for message size to compress the message. | Set the value to 64330 bytes for good message compression. | -| Unit | bytes | -| Value Range | 0: already compress; >0: compress when message exceeds it; -1: always uncompress | -| Default Value | -1 | - -### compressColData - -| Attribute | Description | -| -------- | --------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for size of column data to trigger compression for the query result | -| Unit | bytes | -| Value Range | 0: always compress; >0: only compress when the size of any column data exceeds the threshold; -1: always uncompress | -| Default Value | -1 | -| Default Value | -1 | -| Note | available from version 2.3.0.0 | | - -## Continuous Query Parameters | - -### minSlidingTime - -| Attribute | Description | -| ------------- | -------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | Minimum sliding time of time window | -| Unit | millisecond or microsecond , depending on time precision | -| Value Range | 10-1000000 | -| Default Value | 10 | - -### minIntervalTime - -| Attribute | Description | -| ------------- | --------------------------- | -| Applicable | Server Only | -| Meaning | Minimum size of time window | -| Unit | millisecond | -| Value Range | 1-1000000 | -| Default Value | 10 | - -:::info -To prevent system resource from being exhausted by multiple concurrent streams, a random delay is applied on each stream automatically. `maxFirstStreamCompDelay` is the maximum delay time before a continuous query is started the first time. `streamCompDelayRatio` is the ratio for calculating delay time, with the size of the time window as base. `maxStreamCompDelay` is the maximum delay time. The actual delay time is a random time not bigger than `maxStreamCompDelay`. If a continuous query fails, `retryStreamComDelay` is the delay time before retrying it, also not bigger than `maxStreamCompDelay`. - -::: - ## Log Parameters ### logDir @@ -665,6 +598,18 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Value Range | 0: not consistent; 1: consistent. | | Default | 1 | +## Compress Parameters + +### compressMsgSize + +| Attribute | Description | +| -------- | ----------------------------- | +| Applicable | Both Client and Server side | +| Meaning | Whether RPC message is compressed | +| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | +| Default | -1 | + + ## Other Parameters ### enableCoreFile @@ -686,172 +631,60 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Value Range | 0: disable UDF; 1: enabled UDF | | Default Value | 1 | -## Parameter Comparison of TDengine 2.x and 3.0 -| # | **Parameter** | **In 2.x** | **In 3.0** | -| --- | :-----------------: | --------------- | --------------- | -| 1 | firstEp | Yes | Yes | -| 2 | secondEp | Yes | Yes | -| 3 | fqdn | Yes | Yes | -| 4 | serverPort | Yes | Yes | -| 5 | maxShellConns | Yes | Yes | -| 6 | monitor | Yes | Yes | -| 7 | monitorFqdn | No | Yes | -| 8 | monitorPort | No | Yes | -| 9 | monitorInterval | Yes | Yes | -| 10 | monitorMaxLogs | No | Yes | -| 11 | monitorComp | No | Yes | -| 12 | telemetryReporting | Yes | Yes | -| 13 | telemetryInterval | No | Yes | -| 14 | telemetryServer | No | Yes | -| 15 | telemetryPort | No | Yes | -| 16 | queryPolicy | No | Yes | -| 17 | querySmaOptimize | No | Yes | -| 18 | queryRsmaTolerance | No | Yes | -| 19 | queryBufferSize | Yes | Yes | -| 20 | maxNumOfDistinctRes | Yes | Yes | -| 21 | minSlidingTime | Yes | Yes | -| 22 | minIntervalTime | Yes | Yes | -| 23 | countAlwaysReturnValue | Yes | Yes | -| 24 | dataDir | Yes | Yes | -| 25 | minimalDataDirGB | Yes | Yes | -| 26 | supportVnodes | No | Yes | -| 27 | tempDir | Yes | Yes | -| 28 | minimalTmpDirGB | Yes | Yes | -| 29 | compressMsgSize | Yes | Yes | -| 30 | compressColData | Yes | Yes | -| 31 | smlChildTableName | Yes | Yes | -| 32 | smlTagName | Yes | Yes | -| 33 | smlDataFormat | No | Yes | -| 34 | statusInterval | Yes | Yes | -| 35 | shellActivityTimer | Yes | Yes | -| 36 | transPullupInterval | No | Yes | -| 37 | mqRebalanceInterval | No | Yes | -| 38 | ttlUnit | No | Yes | -| 39 | ttlPushInterval | No | Yes | -| 40 | numOfTaskQueueThreads | No | Yes | -| 41 | numOfRpcThreads | No | Yes | -| 42 | numOfCommitThreads | Yes | Yes | -| 43 | numOfMnodeReadThreads | No | Yes | -| 44 | numOfVnodeQueryThreads | No | Yes | -| 45 | numOfVnodeStreamThreads | No | Yes | -| 46 | numOfVnodeFetchThreads | No | Yes | -| 47 | numOfVnodeRsmaThreads | No | Yes | -| 48 | numOfQnodeQueryThreads | No | Yes | -| 49 | numOfQnodeFetchThreads | No | Yes | -| 50 | numOfSnodeSharedThreads | No | Yes | -| 51 | numOfSnodeUniqueThreads | No | Yes | -| 52 | rpcQueueMemoryAllowed | No | Yes | -| 53 | logDir | Yes | Yes | -| 54 | minimalLogDirGB | Yes | Yes | -| 55 | numOfLogLines | Yes | Yes | -| 56 | asyncLog | Yes | Yes | -| 57 | logKeepDays | Yes | Yes | -| 60 | debugFlag | Yes | Yes | -| 61 | tmrDebugFlag | Yes | Yes | -| 62 | uDebugFlag | Yes | Yes | -| 63 | rpcDebugFlag | Yes | Yes | -| 64 | jniDebugFlag | Yes | Yes | -| 65 | qDebugFlag | Yes | Yes | -| 66 | cDebugFlag | Yes | Yes | -| 67 | dDebugFlag | Yes | Yes | -| 68 | vDebugFlag | Yes | Yes | -| 69 | mDebugFlag | Yes | Yes | -| 70 | wDebugFlag | Yes | Yes | -| 71 | sDebugFlag | Yes | Yes | -| 72 | tsdbDebugFlag | Yes | Yes | -| 73 | tqDebugFlag | No | Yes | -| 74 | fsDebugFlag | Yes | Yes | -| 75 | udfDebugFlag | No | Yes | -| 76 | smaDebugFlag | No | Yes | -| 77 | idxDebugFlag | No | Yes | -| 78 | tdbDebugFlag | No | Yes | -| 79 | metaDebugFlag | No | Yes | -| 80 | timezone | Yes | Yes | -| 81 | locale | Yes | Yes | -| 82 | charset | Yes | Yes | -| 83 | udf | Yes | Yes | -| 84 | enableCoreFile | Yes | Yes | -| 85 | arbitrator | Yes | No | -| 86 | numOfThreadsPerCore | Yes | No | -| 87 | numOfMnodes | Yes | No | -| 88 | vnodeBak | Yes | No | -| 89 | balance | Yes | No | -| 90 | balanceInterval | Yes | No | -| 91 | offlineThreshold | Yes | No | -| 92 | role | Yes | No | -| 93 | dnodeNopLoop | Yes | No | -| 94 | keepTimeOffset | Yes | No | -| 95 | rpcTimer | Yes | No | -| 96 | rpcMaxTime | Yes | No | -| 97 | rpcForceTcp | Yes | No | -| 98 | tcpConnTimeout | Yes | No | -| 99 | syncCheckInterval | Yes | No | -| 100 | maxTmrCtrl | Yes | No | -| 101 | monitorReplica | Yes | No | -| 102 | smlTagNullName | Yes | No | -| 103 | keepColumnName | Yes | No | -| 104 | ratioOfQueryCores | Yes | No | -| 105 | maxStreamCompDelay | Yes | No | -| 106 | maxFirstStreamCompDelay | Yes | No | -| 107 | retryStreamCompDelay | Yes | No | -| 108 | streamCompDelayRatio | Yes | No | -| 109 | maxVgroupsPerDb | Yes | No | -| 110 | maxTablesPerVnode | Yes | No | -| 111 | minTablesPerVnode | Yes | No | -| 112 | tableIncStepPerVnode | Yes | No | -| 113 | cache | Yes | No | -| 114 | blocks | Yes | No | -| 115 | days | Yes | No | -| 116 | keep | Yes | No | -| 117 | minRows | Yes | No | -| 118 | maxRows | Yes | No | -| 119 | quorum | Yes | No | -| 120 | comp | Yes | No | -| 121 | walLevel | Yes | No | -| 122 | fsync | Yes | No | -| 123 | replica | Yes | No | -| 124 | partitions | Yes | No | -| 125 | quorum | Yes | No | -| 126 | update | Yes | No | -| 127 | cachelast | Yes | No | -| 128 | maxSQLLength | Yes | No | -| 129 | maxWildCardsLength | Yes | No | -| 130 | maxRegexStringLen | Yes | No | -| 131 | maxNumOfOrderedRes | Yes | No | -| 132 | maxConnections | Yes | No | -| 133 | mnodeEqualVnodeNum | Yes | No | -| 134 | http | Yes | No | -| 135 | httpEnableRecordSql | Yes | No | -| 136 | httpMaxThreads | Yes | No | -| 137 | restfulRowLimit | Yes | No | -| 138 | httpDbNameMandatory | Yes | No | -| 139 | httpKeepAlive | Yes | No | -| 140 | enableRecordSql | Yes | No | -| 141 | maxBinaryDisplayWidth | Yes | No | -| 142 | stream | Yes | No | -| 143 | retrieveBlockingModel | Yes | No | -| 144 | tsdbMetaCompactRatio | Yes | No | -| 145 | defaultJSONStrType | Yes | No | -| 146 | walFlushSize | Yes | No | -| 147 | keepTimeOffset | Yes | No | -| 148 | flowctrl | Yes | No | -| 149 | slaveQuery | Yes | No | -| 150 | adjustMaster | Yes | No | -| 151 | topicBinaryLen | Yes | No | -| 152 | telegrafUseFieldNum | Yes | No | -| 153 | deadLockKillQuery | Yes | No | -| 154 | clientMerge | Yes | No | -| 155 | sdbDebugFlag | Yes | No | -| 156 | odbcDebugFlag | Yes | No | -| 157 | httpDebugFlag | Yes | No | -| 158 | monDebugFlag | Yes | No | -| 159 | cqDebugFlag | Yes | No | -| 160 | shortcutFlag | Yes | No | -| 161 | probeSeconds | Yes | No | -| 162 | probeKillSeconds | Yes | No | -| 163 | probeInterval | Yes | No | -| 164 | lossyColumns | Yes | No | -| 165 | fPrecision | Yes | No | -| 166 | dPrecision | Yes | No | -| 167 | maxRange | Yes | No | -| 168 | range | Yes | No | + +## 3.0 Parameters + +| # | **参数** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | firstEp | Yes | Yes | | +| 2 | secondEp | Yes | Yes | | +| 3 | fqdn | Yes | Yes | | +| 4 | serverPort | Yes | Yes | | +| 5 | maxShellConns | Yes | Yes | | +| 6 | monitor | Yes | Yes | | +| 7 | monitorFqdn | No | Yes | | +| 8 | monitorPort | No | Yes | | +| 9 | monitorInterval | Yes | Yes | | +| 10 | queryPolicy | No | Yes | | +| 11 | querySmaOptimize | No | Yes | | +| 12 | maxNumOfDistinctRes | Yes | Yes | | +| 15 | countAlwaysReturnValue | Yes | Yes | | +| 16 | dataDir | Yes | Yes | | +| 17 | minimalDataDirGB | Yes | Yes | | +| 18 | supportVnodes | No | Yes | | +| 19 | tempDir | Yes | Yes | | +| 20 | minimalTmpDirGB | Yes | Yes | | +| 21 | smlChildTableName | Yes | Yes | | +| 22 | smlTagName | Yes | Yes | | +| 23 | smlDataFormat | No | Yes | | +| 24 | statusInterval | Yes | Yes | | +| 25 | logDir | Yes | Yes | | +| 26 | minimalLogDirGB | Yes | Yes | | +| 27 | numOfLogLines | Yes | Yes | | +| 28 | asyncLog | Yes | Yes | | +| 29 | logKeepDays | Yes | Yes | | +| 30 | debugFlag | Yes | Yes | | +| 31 | tmrDebugFlag | Yes | Yes | | +| 32 | uDebugFlag | Yes | Yes | | +| 33 | rpcDebugFlag | Yes | Yes | | +| 34 | jniDebugFlag | Yes | Yes | | +| 35 | qDebugFlag | Yes | Yes | | +| 36 | cDebugFlag | Yes | Yes | | +| 37 | dDebugFlag | Yes | Yes | | +| 38 | vDebugFlag | Yes | Yes | | +| 39 | mDebugFlag | Yes | Yes | | +| 40 | wDebugFlag | Yes | Yes | | +| 41 | sDebugFlag | Yes | Yes | | +| 42 | tsdbDebugFlag | Yes | Yes | | +| 43 | tqDebugFlag | No | Yes | | +| 44 | fsDebugFlag | Yes | Yes | | +| 45 | udfDebugFlag | No | Yes | | +| 46 | smaDebugFlag | No | Yes | | +| 47 | idxDebugFlag | No | Yes | | +| 48 | tdbDebugFlag | No | Yes | | +| 49 | metaDebugFlag | No | Yes | | +| 50 | timezone | Yes | Yes | | +| 51 | locale | Yes | Yes | | +| 52 | charset | Yes | Yes | | +| 53 | udf | Yes | Yes | | +| 54 | enableCoreFile | Yes | Yes | | diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 6720af8bf81ea2f4fce415a54847453f578ababf..3e8f7c295d7b6ab81ecb678865ed3e4eff81b2c5 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -76,7 +76,7 @@ Development: false ### Install from source code ``` -git clone https://github.com:taosdata/kafka-connect-tdengine.git +git clone https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine mvn clean package unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 32bdc21e7c0552f5f891b38f806e48efb6c419ac..e33970c407aa8c53c72eb32cf8418cdb709aacdb 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.2.0 + + + ## 3.0.1.8 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 7126b5a997043231d1cf93d633b8cd71e5f6275e..f2212bb2d4f2a692a1285f1869c7df61cc9cc374 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.3.2 + + + ## 2.3.0 diff --git a/docs/examples/csharp/wsConnect/Program.cs b/docs/examples/csharp/wsConnect/Program.cs index f9a56c842fcfe1f0a032cca70460b4f84732cf18..a534bb8a6582be3b8da9ddd73daf39717909b40b 100644 --- a/docs/examples/csharp/wsConnect/Program.cs +++ b/docs/examples/csharp/wsConnect/Program.cs @@ -5,22 +5,24 @@ namespace Examples { public class WSConnExample { - static void Main(string[] args) + static int Main(string[] args) { string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); if (wsConn == IntPtr.Zero) { - throw new Exception("get WS connection failed"); + Console.WriteLine("get WS connection failed"); + return -1; } else { Console.WriteLine("Establish connect success."); + // close connection. + LibTaosWS.WSClose(wsConn); } - // close connection. - LibTaosWS.WSClose(wsConn); + return 0; } } -} \ No newline at end of file +} diff --git a/docs/examples/csharp/wsInsert/Program.cs b/docs/examples/csharp/wsInsert/Program.cs index 1f2d0a67253f478913f7eb0093b732a1a0c0f541..7fa6af805c5f6fc136c89c903e20636eaaa0a0fc 100644 --- a/docs/examples/csharp/wsInsert/Program.cs +++ b/docs/examples/csharp/wsInsert/Program.cs @@ -5,7 +5,7 @@ namespace Examples { public class WSInsertExample { - static void Main(string[] args) + static int Main(string[] args) { string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); @@ -13,7 +13,8 @@ namespace Examples // Assert if connection is validate if (wsConn == IntPtr.Zero) { - throw new Exception("get WS connection failed"); + Console.WriteLine("get WS connection failed"); + return -1; } else { @@ -36,6 +37,8 @@ namespace Examples // close connection. LibTaosWS.WSClose(wsConn); + + return 0; } static void ValidInsert(string desc, IntPtr wsRes) @@ -43,7 +46,7 @@ namespace Examples int code = LibTaosWS.WSErrorNo(wsRes); if (code != 0) { - throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); + Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); } else { @@ -55,4 +58,4 @@ namespace Examples } // Establish connect success. // create table success affect 0 rows, cost 3717542 nanoseconds -// insert data success affect 8 rows, cost 2613637 nanoseconds \ No newline at end of file +// insert data success affect 8 rows, cost 2613637 nanoseconds diff --git a/docs/examples/csharp/wsQuery/Program.cs b/docs/examples/csharp/wsQuery/Program.cs index a220cae9032bb2633f0514cbbd3f859a5f1a1f17..8ee900a05aaef527da83e88340057014a17d9053 100644 --- a/docs/examples/csharp/wsQuery/Program.cs +++ b/docs/examples/csharp/wsQuery/Program.cs @@ -7,13 +7,14 @@ namespace Examples { public class WSQueryExample { - static void Main(string[] args) + static int Main(string[] args) { string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); if (wsConn == IntPtr.Zero) { - throw new Exception("get WS connection failed"); + Console.WriteLine("get WS connection failed"); + return -1; } else { @@ -28,7 +29,9 @@ namespace Examples int code = LibTaosWS.WSErrorNo(wsRes); if (code != 0) { - throw new Exception($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); + Console.WriteLine($"execute SQL failed: reason: {LibTaosWS.WSErrorStr(wsRes)}, code:{code}"); + LibTaosWS.WSFreeResult(wsRes); + return -1; } // get meta data @@ -58,6 +61,8 @@ namespace Examples // close connection. LibTaosWS.WSClose(wsConn); + + return 0; } } } @@ -71,4 +76,4 @@ namespace Examples // 1538548685000 | 10.3 | 219 | 0.31 | California.SanFrancisco | 2 | // 1538548695000 | 12.6 | 218 | 0.33 | California.SanFrancisco | 2 | // 1538548696800 | 12.3 | 221 | 0.31 | California.SanFrancisco | 2 | -// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 | \ No newline at end of file +// 1538548696650 | 10.3 | 218 | 0.25 | California.SanFrancisco | 3 | diff --git a/docs/examples/csharp/wsStmt/Program.cs b/docs/examples/csharp/wsStmt/Program.cs index 8af807ec39d72651c48ab82b79db7f4c5049225f..f8673357db6caef676294fa3d505f5bd4c2872c8 100644 --- a/docs/examples/csharp/wsStmt/Program.cs +++ b/docs/examples/csharp/wsStmt/Program.cs @@ -7,7 +7,7 @@ namespace Examples { public class WSStmtExample { - static void Main(string[] args) + static int Main(string[] args) { const string DSN = "ws://root:taosdata@127.0.0.1:6041/test"; const string table = "meters"; @@ -21,7 +21,8 @@ namespace Examples IntPtr wsConn = LibTaosWS.WSConnectWithDSN(DSN); if (wsConn == IntPtr.Zero) { - throw new Exception($"get WS connection failed"); + Console.WriteLine($"get WS connection failed"); + return -1; } else { @@ -66,18 +67,20 @@ namespace Examples } else { - throw new Exception("Init STMT failed..."); + Console.WriteLine("Init STMT failed..."); } // close connection. LibTaosWS.WSClose(wsConn); + + return 0; } static void ValidStmtStep(int code, IntPtr wsStmt, string desc) { if (code != 0) { - throw new Exception($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}"); + Console.WriteLine($"{desc} failed,reason: {LibTaosWS.WSErrorStr(wsStmt)}, code: {code}"); } else { @@ -92,4 +95,4 @@ namespace Examples // WSStmtBindParamBatch success... // WSStmtAddBatch success... // WSStmtExecute success... -// WS STMT insert 5 rows... \ No newline at end of file +// WS STMT insert 5 rows... diff --git a/docs/examples/python/connection_usage_native_reference.py b/docs/examples/python/connection_usage_native_reference.py index 4803511e427bf4d906fd3a14ff6faf5a000da96c..a7179b4cf859eb440b535a797eeb8e2be1e33589 100644 --- a/docs/examples/python/connection_usage_native_reference.py +++ b/docs/examples/python/connection_usage_native_reference.py @@ -8,7 +8,7 @@ conn.execute("CREATE DATABASE test") # change database. same as execute "USE db" conn.select_db("test") conn.execute("CREATE STABLE weather(ts TIMESTAMP, temperature FLOAT) TAGS (location INT)") -affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m 24.4)") +affected_row: int = conn.execute("INSERT INTO t1 USING weather TAGS(1) VALUES (now, 23.5) (now+1m, 23.5) (now+2m, 24.4)") print("affected_row", affected_row) # output: # affected_row 3 diff --git a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx index 89818409c5032166e77a50f07ea1b54bd66617cb..e1fd3dacc8d91ea4ce3efde6f71a645cc65140ea 100644 --- a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,6 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 :::note - 对于 JSON 格式协议，TDengine 并不会自动把所有标签转成 NCHAR 类型， 字符串将将转为 NCHAR 类型， 数值将同样转换为 DOUBLE 类型。 -- TDengine 只接收 JSON **数组格式**的字符串，即使一行数据也需要转换成数组形式。 - 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下：假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同，但是后面的 tag_set 不同，则使用第一行自动建表时指定的 tag_set，其他的行会忽略）。 ::: diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 1ee59d2df429f2bd64b1dfdcc2ef81f12b1272d0..fc6dc571383b8b003ee6e8c8dce73fa7645978e0 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -68,39 +68,38 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 ### 安装连接器 - + - 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - 仓库，且各大仓库都已同步。 +目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 仓库，且各大仓库都已同步。 - - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [maven.aliyun](https://maven.aliyun.com/mvn/search) +- [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [maven.aliyun](https://maven.aliyun.com/mvn/search) - Maven 项目中，在 pom.xml 中添加以下依赖： +Maven 项目中，在 pom.xml 中添加以下依赖： - ```xml-dtd - - com.taosdata.jdbc - taos-jdbcdriver - 3.0.0 - - ``` +```xml-dtd + + com.taosdata.jdbc + taos-jdbcdriver + 3.0.0 + +``` - - + + - 可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector +可以通过下载 TDengine 的源码，自己编译最新版本的 Java connector - ```shell - git clone https://github.com/taosdata/taos-connector-jdbc.git - cd taos-connector-jdbc - mvn clean install -Dmaven.test.skip=true - ``` +```shell +git clone https://github.com/taosdata/taos-connector-jdbc.git +cd taos-connector-jdbc +mvn clean install -Dmaven.test.skip=true +``` - 编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 +编译后，在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包，并自动将编译的 jar 文件放在本地的 Maven 仓库中。 - + ## 建立连接 @@ -111,125 +110,117 @@ TDengine 的 JDBC URL 规范格式为： 对于建立连接，原生连接与 REST 连接有细微不同。 - + - ```java - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.TSDBDriver"); +String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL - 中指定用户名（user）为 root，密码（password）为 taosdata。 +以上示例，使用了 JDBC 原生连接的 TSDBDriver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL +中指定用户名（user）为 root，密码（password）为 taosdata。 - **注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。 +**注意**：使用 JDBC 原生连接，taos-jdbcdriver 需要依赖客户端驱动（Linux 下是 libtaos.so；Windows 下是 taos.dll；macOS 下是 libtaos.dylib）。 - url 中的配置参数如下： +url 中的配置参数如下： - - user：登录 TDengine 用户名，默认值 'root'。 - - password：用户登录密码，默认值 'taosdata'。 - - cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 - - charset：客户端使用的字符集，默认值为系统字符集。 - - locale：客户端语言环境，默认值系统当前 locale。 - - timezone：客户端使用的时区，默认值为系统当前时区。 - - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 - - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL - 后的任何语句。默认值为：false。 +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- cfgdir：客户端配置文件目录路径，Linux OS 上默认值 `/etc/taos`，Windows OS 上默认值 `C:/TDengine/cfg`。 +- charset：客户端使用的字符集，默认值为系统字符集。 +- locale：客户端语言环境，默认值系统当前 locale。 +- timezone：客户端使用的时区，默认值为系统当前时区。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 +- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false：不再执行失败 SQL 后的任何语句。默认值为：false。 - JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 +JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 - **使用 TDengine 客户端驱动配置文件建立连接 ** +**使用 TDengine 客户端驱动配置文件建立连接 ** - 当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示： +当使用 JDBC 原生连接连接 TDengine 集群时，可以使用 TDengine 客户端驱动配置文件，在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示： - 1. 在 Java 应用中不指定 hostname 和 port +1. 在 Java 应用中不指定 hostname 和 port - ```java - public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; - } - ``` +```java +public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; +} +``` - 2. 在配置文件中指定 firstEp 和 secondEp +2. 在配置文件中指定 firstEp 和 secondEp - ```shell - # first fully qualified domain name (FQDN) for TDengine system - firstEp cluster_node1:6030 +```shell +# first fully qualified domain name (FQDN) for TDengine system +firstEp cluster_node1:6030 - # second fully qualified domain name (FQDN) for TDengine system, for cluster only - secondEp cluster_node2:6030 +# second fully qualified domain name (FQDN) for TDengine system, for cluster only +secondEp cluster_node2:6030 - # default system charset - # charset UTF-8 +# default system charset +# charset UTF-8 - # system locale - # locale en_US.UTF-8 - ``` +# system locale +# locale en_US.UTF-8 +``` - 以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp - 连接集群。 +以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。 - TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 +TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 - > **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 - C://TDengine/cfg/taos.cfg。 +> **注意**：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 - - + + - ```java - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 - root，密码（password）为 taosdata。 +以上示例，使用了 JDBC REST 连接的 RestfulDriver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 - 使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： +使用 JDBC REST 连接，不需要依赖客户端驱动。与 JDBC 原生连接相比，仅需要： - 1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； - 2. jdbcUrl 以“jdbc:TAOS-RS://”开头； - 3. 使用 6041 作为连接端口。 +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； +2. jdbcUrl 以“jdbc:TAOS-RS://”开头； +3. 使用 6041 作为连接端口。 - url 中的配置参数如下： +url 中的配置参数如下： - - user：登录 TDengine 用户名，默认值 'root'。 - - password：用户登录密码，默认值 'taosdata'。 - - batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST - 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 - - charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 - - batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL - 后的任何语句。默认值为：false。 - - httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 - - httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 - - messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 - - useSSL: 连接中是否使用 SSL。 +- user：登录 TDengine 用户名，默认值 'root'。 +- password：用户登录密码，默认值 'taosdata'。 +- batchfetch: true：在执行查询时批量拉取结果集；false：逐行拉取结果集。默认值为：false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP，WebSocket 可以使 JDBC REST 连接支持大数据量查询，并提升查询性能。 +- charset: 当开启批量拉取数据时，指定解析字符串数据的字符集。 +- batchErrorIgnore：true：在执行 Statement 的 executeBatch 时，如果中间有一条 SQL 执行失败，继续执行下面的 SQL 了。false：不再执行失败 SQL 后的任何语句。默认值为：false。 +- httpConnectTimeout: 连接超时时间，单位 ms， 默认值为 5000。 +- httpSocketTimeout: socket 超时时间，单位 ms，默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms， 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 - **注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 +**注意**：部分配置项（比如：locale、timezone）在 REST 连接中不生效。 - :::note +:::note - - 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如： +- 与原生连接方式不同，REST 接口是无状态的。在使用 JDBC REST 连接时，需要在 SQL 中指定表、超级表的数据库名称。例如： - ```sql - INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); - ``` +```sql +INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); +``` - - 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 - jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) - tags('California.SanFrancisco') values(now, 24.6); +- 如果在 url 中指定了 dbname，那么，JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url，在 SQL 中不需要指定 dbname。例如：url 为 jdbc:TAOS-RS://127.0.0.1:6041/test，那么，可以执行 sql：insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); - ::: +::: - + ### 指定 URL 和 Properties 获取连接 @@ -890,8 +881,10 @@ public static void main(String[] args) throws Exception { | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | +| 3.0.3 | 修复 REST 连接在 jdk17+ 版本时间戳解析错误问题 | | 3.0.1 - 3.0.2 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译，JDK 8 环境下建议使用 3.0.2 版本 | | 3.0.0 | 支持 TDengine 3.0 | +| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | | 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | | 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | @@ -928,7 +921,7 @@ public static void main(String[] args) throws Exception { **原因**：taos-jdbcdriver 3.0.1 版本需要在 JDK 11+ 环境使用。 -**解决方法**： 更换 taos-jdbcdriver 3.0.2 版本。 +**解决方法**： 更换 taos-jdbcdriver 3.0.2+ 版本。 其它问题请参考 [FAQ](../../../train-faq/faq) diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index eecf564b905ddbf1e930acbb196cfb80869b67e9..a28a7cd66c1cc7598d473903eb95df5192d933d2 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -59,10 +59,10 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | -| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | +| **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | +| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | | **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | -| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | +| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::warning diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 3918f240b4f9037db92af80df26dd471868b29de..232a6c7326ba10740f9e8606254ae97591dc9084 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -27,7 +27,6 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} @@ -61,9 +60,6 @@ database_option: { - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。 - REPLICA：表示数据库副本数，取值为 1 或 3，默认为 1。在集群中使用，副本数必须小于或等于 DNODE 的数目。 - RETENTIONS：表示数据的聚合周期和保存时长，如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒，原始数据保存 7 天；按 1 分钟聚合的数据保存 21 天；按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 -- STRICT：表示数据同步的一致性要求，默认为 off。 - - on 表示强一致，即运行标准的 raft 协议，半数提交返回成功。 - - off 表示弱一致，本地提交即返回成功。 - WAL_LEVEL：WAL 级别，默认为 1。 - 1：写 WAL，但不执行 fsync。 - 2：写 WAL，而且执行 fsync。 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index a8a1edc9a69990ee496881660be6a0b1b5ed16b6..afe90b8a93f5ef6f7b36d43619fb5ba222180ca3 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -533,7 +533,12 @@ TIMEDIFF(expr1, expr2 [, time_unit]) #### TIMETRUNCATE ```sql -TIMETRUNCATE(expr, time_unit) +TIMETRUNCATE(expr, time_unit [, ignore_timezone]) + +ignore_timezone: { + 0 + | 1 +} ``` **功能说明**：将时间戳按照指定时间单位 time_unit 进行截断。 @@ -549,6 +554,11 @@ TIMETRUNCATE(expr, time_unit) 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - 输入包含不符合时间日期格式的字符串则返回 NULL。 +- 当使用 1d 作为时间单位对时间戳进行截断时， 可通过设置 ignore_timezone 参数指定返回结果的显示是否忽略客户端时区的影响。 + 例如客户端所配置时区为 UTC+0800, 则 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) 返回结果为 '2020-01-01 08:00:00'。 + 而使用 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) 设置忽略时区时，返回结果为 '2020-01-01 00:00:00' + ignore_timezone 如果忽略的话，则默认值为 1 。 + #### TIMEZONE @@ -1085,7 +1095,7 @@ ignore_negative: { ```sql DIFF(expr [, ignore_negative]) - + ignore_negative: { 0 | 1 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 932ad30b1a949d172d81819f2432daa42ce331c8..a70d559a860c4e6ec44cbd13c34f7f306ab452cb 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -8,7 +8,7 @@ description: 流式计算的相关 SQL 的详细语法 ## 创建流式计算 ```sql -CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name AS subquery +CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery stream_options: { TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] WATERMARK time @@ -28,6 +28,9 @@ subquery: SELECT select_list 支持会话窗口、状态窗口与滑动窗口，其中，会话窗口与状态窗口搭配超级表时必须与partition by tbname一起使用 + +subtable 子句定义了流式计算中创建的子表的命名规则，详见 流式计算的 partition 部分。 + ```sql window_clause: { SESSION(ts_col, tol_val) @@ -49,11 +52,43 @@ SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL( ## 流式计算的 partition -可以使用 PARTITION BY TBNAME 或 PARTITION BY tag，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。 +可以使用 PARTITION BY TBNAME，tag，普通列或者表达式，对一个流进行多分区的计算，每个分区的时间线与时间窗口是独立的，会各自聚合，并写入到目的表中的不同子表。 + +不带 PARTITION BY 子句时，所有的数据将写入到一张子表。 + +在创建流时不使用 SUBTABLE 子句时，流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。 + +若创建流的语句中包含 SUBTABLE 子句，用户可以为每个 partition 对应的子表生成自定义的表名，例如： + +```sql +CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); +``` + +PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名。 + +注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。 + +## 流式计算读取历史数据 -不带 PARTITION BY 选项时，所有的数据将写入到一张子表。 +正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。例如： + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s) +``` + +结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020年1月30日）之后的数据 + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s) +``` + +再如，仅处理某时间段内的数据，结束时间可以是未来时间 + +```sql +create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' and ts < '2023-01-01' interval(10s) +``` -流式计算创建的超级表有唯一的 tag 列 groupId，每个 partition 会被分配唯一 groupId。与 schemaless 写入一致，我们通过 MD5 计算子表名，并自动创建它。 +如果该流任务已经彻底过期，并且您不再想让它检测或处理数据，您可以手动删除它，被计算出的数据仍会被保留。 ## 删除流式计算 diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 4e1e7b7ace09f7cb43e5049ebefe21a24525b595..0909ddf639b621b39af26e7bd378f8bf7e3b99e7 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -59,6 +59,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +101,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +112,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +134,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -195,6 +199,7 @@ AllowWebSockets - `precision` TDengine 使用的时间精度 - `u` TDengine 用户名 - `p` TDengine 密码 +- `ttl` 自动创建的子表生命周期，以子表的第一条数据的 TTL 参数为准，不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数。 注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。 示例： curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 76dd5f12d89b223292a8c868fd66135ec204319b..9091e71d1f9b118660d5f75ef9be5627143393b0 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -204,6 +204,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-a/--replica ** : 创建数据库时指定其副本数，默认值为 1 。 +- ** -k/--keep-trying ** : 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。 + +- ** -z/--trying-interval ** : 失败重试间隔时间，单位为毫秒，仅在 -k 指定重试后有效。需使用 v3.0.9 以上版本。 + - **-V/--version** : 显示版本信息并退出。不能与其它参数混用。 @@ -231,6 +235,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) 插入场景下 `filetype` 必须设置为 `insert`，该参数及其它通用参数详见[通用配置参数](#通用配置参数) +- ** keep_trying ** : 失败后进行重试的次数，默认不重试。需使用 v3.0.9 以上版本。 + +- ** trying_interval ** : 失败重试间隔时间，单位为毫秒，仅在 keep_trying 指定重试后有效。需使用 v3.0.9 以上版本。 + #### 数据库相关配置参数 创建数据库时的相关参数在 json 配置文件中的 `dbinfo` 中配置，个别具体参数如下。其余参数均与 TDengine 中 `create database` 时所指定的数据库参数相对应，详见[../../taos-sql/database] diff --git a/docs/zh/14-reference/06-taosdump.md b/docs/zh/14-reference/06-taosdump.md index 625499a94926ac3f86e4d34976c70a0bfe7b0954..8a031d147377713d0478d419798240f6f1194377 100644 --- a/docs/zh/14-reference/06-taosdump.md +++ b/docs/zh/14-reference/06-taosdump.md @@ -22,7 +22,7 @@ taosdump 是一个逻辑备份工具，它不应被用于备份任何原始数 taosdump 有两种安装方式: -- 安装 taosTools 官方安装包, 请从[所有下载链接](https://www.taosdata.com/all-downloads)页面找到 taosTools 并下载安装。 +- 安装 taosTools 官方安装包, 请从[发布历史页面](https://docs.taosdata.com/releases/tools/)页面找到 taosTools 并下载安装。 - 单独编译 taos-tools 并安装, 详情请参考 [taos-tools](https://github.com/taosdata/taos-tools) 仓库。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 145c5eed935568f013d4451cd590c8b745b165c5..bc62a536e535b4de642d66849de7e6b10373a191 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -134,15 +134,6 @@ taos --dump-config | 取值范围 | 1-200000 | | 缺省值 | 30 | -### telemetryReporting - -| 属性 | 说明 | -| -------- | ---------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 是否允许 TDengine 采集和上报基本使用信息 | -| 取值范围 | 0：不允许 1：允许 | -| 缺省值 | 1 | - ## 查询相关 ### queryPolicy @@ -191,6 +182,15 @@ taos --dump-config | 取值范围 | 0 表示包含函数名，1 表示不包含函数名。 | | 缺省值 | 0 | +### countAlwaysReturnValue + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | count/hyperloglog函数在数据为空或者NULL的情况下是否返回值 | +| 取值范围 | 0：返回空行，1：返回 0 | +| 缺省值 | 1 | + ## 区域相关 ### timezone @@ -306,12 +306,20 @@ charset 的有效值是 UTF-8。 | 含义 | 数据文件目录，所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | +### tempDir + +| 属性 | 说明 | +| -------- | ------------------------------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | 该参数指定所有系统运行过程中的临时文件生成的目录 | +| 缺省值 | /tmp | + ### minimalTmpDirGB | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 服务端和客户端均适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时，停止写临时文件 | +| 含义 | tempDir 所指定的临时文件目录所需要保留的最小空间 | | 单位 | GB | | 缺省值 | 1.0 | @@ -320,7 +328,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 仅服务端适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时，停止写时序数据 | +| 含义 | dataDir 指定的时序数据存储目录所需要保留的最小 | | 单位 | GB | | 缺省值 | 2.0 | @@ -335,27 +343,7 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0-4096 | | 缺省值 | CPU 核数的 2 倍 | -## 时间相关 - -### statusInterval - -| 属性 | 说明 | -| -------- | --------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | dnode 向 mnode 报告状态间隔 | -| 单位 | 秒 | -| 取值范围 | 1-10 | -| 缺省值 | 1 | - -### shellActivityTimer - -| 属性 | 说明 | -| -------- | --------------------------------- | -| 适用范围 | 服务端和客户端均适用 | -| 含义 | shell 客户端向 mnode 发送心跳间隔 | -| 单位 | 秒 | -| 取值范围 | 1-120 | -| 缺省值 | 3 | +## 时间相关 | ## 性能调优 @@ -367,28 +355,6 @@ charset 的有效值是 UTF-8。 | 含义 | 设置写入线程的最大数量 | | 缺省值 | | -## 压缩相关 - -### compressMsgSize - -| 属性 | 说明 | -| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中，对通讯的消息进行压缩的阈值。如果要压缩消息，建议设置为 64330 字节，即大于 64330 字节的消息体才进行压缩。 | -| 单位 | bytes | -| 取值范围 | `0 `表示对所有的消息均进行压缩 >0: 超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - -### compressColData - -| 属性 | 说明 | -| -------- | --------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中，对服务器端查询结果进行列压缩的阈值。 | -| 单位 | bytes | -| 取值范围 | 0: 对所有查询结果均进行压缩 >0: 查询结果中任意列大小超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - ## 日志相关 ### logDir @@ -613,7 +579,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | schemaless 自定义的子表名 | +| 含义 | schemaless 自定义的子表名的 key | | 类型 | 字符串 | | 缺省值 | 无 | @@ -656,12 +622,18 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不启动；1：启动 | | 缺省值 | 1 | -## 2.X 与 3.0 配置参数对比 +## 压缩参数 -:::note -对于 2.x 版本中适用但在 3.0 版本中废弃的参数，其当前行为会有特别说明 +### compressMsgSize -::: +| 属性 | 说明 | +| -------- | ----------------------------- | +| 适用于 | 服务端和客户端均适用 | +| 含义 | 是否对 RPC 消息进行压缩 | +| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | +| 缺省值 | -1 | + +## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | | --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | @@ -674,159 +646,134 @@ charset 的有效值是 UTF-8。 | 7 | monitorFqdn | 否 | 是 | | | 8 | monitorPort | 否 | 是 | | | 9 | monitorInterval | 是 | 是 | | -| 10 | monitorMaxLogs | 否 | 是 | | -| 11 | monitorComp | 否 | 是 | | -| 12 | telemetryReporting | 是 | 是 | | -| 13 | telemetryInterval | 否 | 是 | | -| 14 | telemetryServer | 否 | 是 | | -| 15 | telemetryPort | 否 | 是 | | -| 16 | queryPolicy | 否 | 是 | | -| 17 | querySmaOptimize | 否 | 是 | | -| 18 | queryRsmaTolerance | 否 | 是 | | -| 19 | queryBufferSize | 是 | 是 | | -| 20 | maxNumOfDistinctRes | 是 | 是 | | -| 21 | minSlidingTime | 是 | 是 | | -| 22 | minIntervalTime | 是 | 是 | | -| 23 | countAlwaysReturnValue | 是 | 是 | | -| 24 | dataDir | 是 | 是 | | -| 25 | minimalDataDirGB | 是 | 是 | | -| 26 | supportVnodes | 否 | 是 | | -| 27 | tempDir | 是 | 是 | | -| 28 | minimalTmpDirGB | 是 | 是 | | -| 29 | compressMsgSize | 是 | 是 | | -| 30 | compressColData | 是 | 是 | | -| 31 | smlChildTableName | 是 | 是 | | -| 32 | smlTagName | 是 | 是 | | -| 33 | smlDataFormat | 否 | 是 | | -| 34 | statusInterval | 是 | 是 | | -| 35 | shellActivityTimer | 是 | 是 | | -| 36 | transPullupInterval | 否 | 是 | | -| 37 | mqRebalanceInterval | 否 | 是 | | -| 38 | ttlUnit | 否 | 是 | | -| 39 | ttlPushInterval | 否 | 是 | | -| 40 | numOfTaskQueueThreads | 否 | 是 | | -| 41 | numOfRpcThreads | 否 | 是 | | -| 42 | numOfCommitThreads | 是 | 是 | | -| 43 | numOfMnodeReadThreads | 否 | 是 | | -| 44 | numOfVnodeQueryThreads | 否 | 是 | | -| 45 | numOfVnodeStreamThreads | 否 | 是 | | -| 46 | numOfVnodeFetchThreads | 否 | 是 | | -| 47 | numOfVnodeRsmaThreads | 否 | 是 | | -| 48 | numOfQnodeQueryThreads | 否 | 是 | | -| 49 | numOfQnodeFetchThreads | 否 | 是 | | -| 50 | numOfSnodeSharedThreads | 否 | 是 | | -| 51 | numOfSnodeUniqueThreads | 否 | 是 | | -| 52 | rpcQueueMemoryAllowed | 否 | 是 | | -| 53 | logDir | 是 | 是 | | -| 54 | minimalLogDirGB | 是 | 是 | | -| 55 | numOfLogLines | 是 | 是 | | -| 56 | asyncLog | 是 | 是 | | -| 57 | logKeepDays | 是 | 是 | | -| 60 | debugFlag | 是 | 是 | | -| 61 | tmrDebugFlag | 是 | 是 | | -| 62 | uDebugFlag | 是 | 是 | | -| 63 | rpcDebugFlag | 是 | 是 | | -| 64 | jniDebugFlag | 是 | 是 | | -| 65 | qDebugFlag | 是 | 是 | | -| 66 | cDebugFlag | 是 | 是 | | -| 67 | dDebugFlag | 是 | 是 | | -| 68 | vDebugFlag | 是 | 是 | | -| 69 | mDebugFlag | 是 | 是 | | -| 70 | wDebugFlag | 是 | 是 | | -| 71 | sDebugFlag | 是 | 是 | | -| 72 | tsdbDebugFlag | 是 | 是 | | -| 73 | tqDebugFlag | 否 | 是 | | -| 74 | fsDebugFlag | 是 | 是 | | -| 75 | udfDebugFlag | 否 | 是 | | -| 76 | smaDebugFlag | 否 | 是 | | -| 77 | idxDebugFlag | 否 | 是 | | -| 78 | tdbDebugFlag | 否 | 是 | | -| 79 | metaDebugFlag | 否 | 是 | | -| 80 | timezone | 是 | 是 | | -| 81 | locale | 是 | 是 | | -| 82 | charset | 是 | 是 | | -| 83 | udf | 是 | 是 | | -| 84 | enableCoreFile | 是 | 是 | | -| 85 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | -| 86 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | -| 87 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | -| 88 | vnodeBak | 是 | 否 | 3.0 行为未知 | -| 89 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 | -| 90 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | -| 91 | offlineThreshold | 是 | 否 | 3.0 行为未知 | -| 92 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | -| 93 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | -| 94 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | -| 95 | rpcTimer | 是 | 否 | 3.0 行为未知 | -| 96 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | -| 97 | rpcForceTcp | 是 | 否 | 默认为 TCP | -| 98 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | -| 99 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | -| 100 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | -| 101 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | -| 102 | smlTagNullName | 是 | 否 | 3.0 行为未知 | -| 103 | keepColumnName | 是 | 否 | 3.0 行为未知 | -| 104 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | -| 105 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 106 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 107 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 108 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | -| 109 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | -| 110 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 111 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 112 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 113 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 114 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 115 | days | 是 | 否 | 由 create db 的参数 duration 取代 | -| 116 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | -| 117 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | -| 118 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | -| 119 | quorum | 是 | 否 | 由 RAFT 协议决定 | -| 120 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | -| 121 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | -| 122 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | -| 123 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | -| 124 | partitions | 是 | 否 | 3.0 行为未知 | -| 125 | update | 是 | 否 | 允许更新部分列 | -| 126 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | -| 127 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | -| 128 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | -| 129 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | -| 130 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | -| 131 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | -| 132 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | -| 133 | http | 是 | 否 | http 服务由 taosAdapter 提供 | -| 134 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | -| 135 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | -| 136 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | -| 137 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | -| 138 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | -| 139 | enableRecordSql | 是 | 否 | 3.0 行为未知 | -| 140 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | -| 141 | stream | 是 | 否 | 默认启用连续查询 | -| 142 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | -| 143 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | -| 144 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | -| 145 | walFlushSize | 是 | 否 | 3.0 行为未知 | -| 146 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | -| 147 | flowctrl | 是 | 否 | 3.0 行为未知 | -| 148 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | -| 149 | adjustMaster | 是 | 否 | 3.0 行为未知 | -| 150 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | -| 151 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | -| 152 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | -| 153 | clientMerge | 是 | 否 | 3.0 行为未知 | -| 154 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 155 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 156 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 157 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 158 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 159 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 160 | probeSeconds | 是 | 否 | 3.0 行为未知 | -| 161 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | -| 162 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 163 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 164 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 165 | dPrecision | 是 | 否 | 3.0 行为未知 | -| 166 | maxRange | 是 | 否 | 3.0 行为未知 | -| 167 | range | 是 | 否 | 3.0 行为未知 | +| 10 | queryPolicy | 否 | 是 | | +| 11 | querySmaOptimize | 否 | 是 | | +| 12 | maxNumOfDistinctRes | 是 | 是 | | +| 15 | countAlwaysReturnValue | 是 | 是 | | +| 16 | dataDir | 是 | 是 | | +| 17 | minimalDataDirGB | 是 | 是 | | +| 18 | supportVnodes | 否 | 是 | | +| 19 | tempDir | 是 | 是 | | +| 20 | minimalTmpDirGB | 是 | 是 | | +| 21 | smlChildTableName | 是 | 是 | | +| 22 | smlTagName | 是 | 是 | | +| 23 | smlDataFormat | 否 | 是 | | +| 24 | statusInterval | 是 | 是 | | +| 25 | logDir | 是 | 是 | | +| 26 | minimalLogDirGB | 是 | 是 | | +| 27 | numOfLogLines | 是 | 是 | | +| 28 | asyncLog | 是 | 是 | | +| 29 | logKeepDays | 是 | 是 | | +| 30 | debugFlag | 是 | 是 | | +| 31 | tmrDebugFlag | 是 | 是 | | +| 32 | uDebugFlag | 是 | 是 | | +| 33 | rpcDebugFlag | 是 | 是 | | +| 34 | jniDebugFlag | 是 | 是 | | +| 35 | qDebugFlag | 是 | 是 | | +| 36 | cDebugFlag | 是 | 是 | | +| 37 | dDebugFlag | 是 | 是 | | +| 38 | vDebugFlag | 是 | 是 | | +| 39 | mDebugFlag | 是 | 是 | | +| 40 | wDebugFlag | 是 | 是 | | +| 41 | sDebugFlag | 是 | 是 | | +| 42 | tsdbDebugFlag | 是 | 是 | | +| 43 | tqDebugFlag | 否 | 是 | | +| 44 | fsDebugFlag | 是 | 是 | | +| 45 | udfDebugFlag | 否 | 是 | | +| 46 | smaDebugFlag | 否 | 是 | | +| 47 | idxDebugFlag | 否 | 是 | | +| 48 | tdbDebugFlag | 否 | 是 | | +| 49 | metaDebugFlag | 否 | 是 | | +| 50 | timezone | 是 | 是 | | +| 51 | locale | 是 | 是 | | +| 52 | charset | 是 | 是 | | +| 53 | udf | 是 | 是 | | +| 54 | enableCoreFile | 是 | 是 | | + +## 2.x->3.0 的废弃参数 + +| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | +| 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | +| 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | +| 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | +| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 | +| 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | +| 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | +| 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | +| 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | +| 10 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | +| 11 | rpcTimer | 是 | 否 | 3.0 行为未知 | +| 12 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | +| 13 | rpcForceTcp | 是 | 否 | 默认为 TCP | +| 14 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | +| 15 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | +| 16 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | +| 17 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | +| 18 | smlTagNullName | 是 | 否 | 3.0 行为未知 | +| 19 | keepColumnName | 是 | 否 | 3.0 行为未知 | +| 20 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | +| 21 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 22 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 23 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 24 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | +| 25 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | +| 26 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 27 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 28 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 29 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 30 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 31 | days | 是 | 否 | 由 create db 的参数 duration 取代 | +| 32 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | +| 33 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | +| 34 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | +| 35 | quorum | 是 | 否 | 由 RAFT 协议决定 | +| 36 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | +| 37 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | +| 38 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | +| 39 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | +| 40 | partitions | 是 | 否 | 3.0 行为未知 | +| 41 | update | 是 | 否 | 允许更新部分列 | +| 42 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | +| 43 | maxSQLLength | 是 | 否 | SQL 上限为 1MB，无需参数控制 | +| 44 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | +| 45 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | +| 46 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | +| 47 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力，详见后面的 Note | +| 48 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | +| 49 | http | 是 | 否 | http 服务由 taosAdapter 提供 | +| 50 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | +| 51 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | +| 52 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | +| 53 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | +| 54 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | +| 55 | enableRecordSql | 是 | 否 | 3.0 行为未知 | +| 56 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | +| 57 | stream | 是 | 否 | 默认启用连续查询 | +| 58 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | +| 59 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | +| 60 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | +| 61 | walFlushSize | 是 | 否 | 3.0 行为未知 | +| 62 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | +| 63 | flowctrl | 是 | 否 | 3.0 行为未知 | +| 64 | slaveQuery | 是 | 否 | 3.0 行为未知： slave vnode 是否能够处理查询？ | +| 65 | adjustMaster | 是 | 否 | 3.0 行为未知 | +| 66 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | +| 67 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | +| 68 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | +| 69 | clientMerge | 是 | 否 | 3.0 行为未知 | +| 70 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 71 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 72 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 73 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 74 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 75 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 76 | probeSeconds | 是 | 否 | 3.0 行为未知 | +| 77 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | +| 78 | probeInterval | 是 | 否 | 3.0 行为未知 | +| 79 | lossyColumns | 是 | 否 | 3.0 行为未知 | +| 80 | fPrecision | 是 | 否 | 3.0 行为未知 | +| 81 | dPrecision | 是 | 否 | 3.0 行为未知 | +| 82 | maxRange | 是 | 否 | 3.0 行为未知 | +| 83 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/20-third-party/11-kafka.md b/docs/zh/20-third-party/11-kafka.md index 1172f4fbc5bcd9f240bd5e2a47108a8791810e76..cc2247f25ebacbc8f5290c2442860f264fad39f1 100644 --- a/docs/zh/20-third-party/11-kafka.md +++ b/docs/zh/20-third-party/11-kafka.md @@ -79,7 +79,7 @@ Development: false ### 从源码安装 ``` -git clone https://github.com:taosdata/kafka-connect-tdengine.git +git clone https://github.com/taosdata/kafka-connect-tdengine.git cd kafka-connect-tdengine mvn clean package unzip -d $CONFLUENT_HOME/share/java/ target/components/packages/taosdata-kafka-connect-tdengine-*.zip diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 7ed9e0c5a018401e2028a1e0786459d4e26f27b6..0ea34829a5f2c28658f5cc41d8a3341efdd0b39d 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,11 +10,14 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.2.0 + + + ## 3.0.1.8 - ## 3.0.1.7 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 67ca3fae67b36e5f08c57440bbaa64ec4f80bf4e..6471826b501996c4ba033a8fcbb08fc4e7333c03 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.3.2 + + + ## 2.3.0 diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index 37edc739e47b88236ee98c3ef0f1185c622abf9d..e14c4e60d9d49a660643c89f4642af7b12c8d18a 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -3,19 +3,13 @@ PROJECT(TDengine) IF (TD_LINUX) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) - # ADD_EXECUTABLE(demo apitest.c) - #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) - #ADD_EXECUTABLE(sml schemaless.c) - #TARGET_LINK_LIBRARIES(sml taos_static trpc tutil pthread ) - #ADD_EXECUTABLE(subscribe subscribe.c) - #TARGET_LINK_LIBRARIES(subscribe taos_static trpc tutil pthread ) - #ADD_EXECUTABLE(epoll epoll.c) - #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) add_executable(tmq "") add_executable(stream_demo "") - add_executable(demoapi "") - add_executable(api_reqid "") + add_executable(schemaless "") + add_executable(prepare "") + add_executable(demo "") + add_executable(asyncdemo "") target_sources(tmq PRIVATE @@ -27,16 +21,25 @@ IF (TD_LINUX) "stream_demo.c" ) - target_sources(demoapi + target_sources(schemaless PRIVATE - "demoapi.c" + "schemaless.c" ) - target_sources(api_reqid + target_sources(prepare PRIVATE - "api_with_reqid_test.c" + "prepare.c" ) + target_sources(demo + PRIVATE + "demo.c" + ) + + target_sources(asyncdemo + PRIVATE + "asyncdemo.c" + ) target_link_libraries(tmq taos_static @@ -46,46 +49,30 @@ IF (TD_LINUX) taos_static ) - target_link_libraries(demoapi + target_link_libraries(schemaless taos_static ) - target_link_libraries(api_reqid + target_link_libraries(prepare taos_static ) - - target_include_directories(tmq - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" - ) - - target_include_directories(stream_demo - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" - ) - - target_include_directories(demoapi - PUBLIC "${TD_SOURCE_DIR}/include/client" - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + target_link_libraries(demo + taos_static ) - target_include_directories(api_reqid - PUBLIC "${TD_SOURCE_DIR}/include/client" - PUBLIC "${TD_SOURCE_DIR}/include/os" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + target_link_libraries(asyncdemo + taos_static ) SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) SET_TARGET_PROPERTIES(stream_demo PROPERTIES OUTPUT_NAME stream_demo) - SET_TARGET_PROPERTIES(demoapi PROPERTIES OUTPUT_NAME demoapi) - SET_TARGET_PROPERTIES(api_reqid PROPERTIES OUTPUT_NAME api_reqid) + SET_TARGET_PROPERTIES(schemaless PROPERTIES OUTPUT_NAME schemaless) + SET_TARGET_PROPERTIES(prepare PROPERTIES OUTPUT_NAME prepare) + SET_TARGET_PROPERTIES(demo PROPERTIES OUTPUT_NAME demo) + SET_TARGET_PROPERTIES(asyncdemo PROPERTIES OUTPUT_NAME asyncdemo) ENDIF () IF (TD_DARWIN) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) - #ADD_EXECUTABLE(demo demo.c) - #TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) - #ADD_EXECUTABLE(epoll epoll.c) - #TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) ENDIF () diff --git a/examples/c/apitest.c b/examples/c/apitest.c deleted file mode 100644 index c179acaf4ef086a702e3130d8ae7effade0f09b9..0000000000000000000000000000000000000000 --- a/examples/c/apitest.c +++ /dev/null @@ -1,1948 +0,0 @@ -// sample code to verify all TDengine API -// to compile: gcc -o apitest apitest.c -ltaos - -#include "cJSON.h" -#include "taoserror.h" - -#include -#include -#include -#include -#include "../../../include/client/taos.h" - -static void prepare_data(TAOS* taos) { - TAOS_RES* result; - result = taos_query(taos, "drop database if exists test;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database test precision 'us';"); - taos_free_result(result); - usleep(100000); - taos_select_db(taos, "test"); - - result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); - taos_free_result(result); - - result = taos_query(taos, "create table t0 using meters tags(0);"); - taos_free_result(result); - result = taos_query(taos, "create table t1 using meters tags(1);"); - taos_free_result(result); - result = taos_query(taos, "create table t2 using meters tags(2);"); - taos_free_result(result); - result = taos_query(taos, "create table t3 using meters tags(3);"); - taos_free_result(result); - result = taos_query(taos, "create table t4 using meters tags(4);"); - taos_free_result(result); - result = taos_query(taos, "create table t5 using meters tags(5);"); - taos_free_result(result); - result = taos_query(taos, "create table t6 using meters tags(6);"); - taos_free_result(result); - result = taos_query(taos, "create table t7 using meters tags(7);"); - taos_free_result(result); - result = taos_query(taos, "create table t8 using meters tags(8);"); - taos_free_result(result); - result = taos_query(taos, "create table t9 using meters tags(9);"); - taos_free_result(result); - - result = taos_query(taos, - "insert into t0 values('2020-01-01 00:00:00.000', 0)" - " ('2020-01-01 00:01:00.000', 0)" - " ('2020-01-01 00:02:00.000', 0)" - " t1 values('2020-01-01 00:00:00.000', 0)" - " ('2020-01-01 00:01:00.000', 0)" - " ('2020-01-01 00:02:00.000', 0)" - " ('2020-01-01 00:03:00.000', 0)" - " t2 values('2020-01-01 00:00:00.000', 0)" - " ('2020-01-01 00:01:00.000', 0)" - " ('2020-01-01 00:01:01.000', 0)" - " ('2020-01-01 00:01:02.000', 0)" - " t3 values('2020-01-01 00:01:02.000', 0)" - " t4 values('2020-01-01 00:01:02.000', 0)" - " t5 values('2020-01-01 00:01:02.000', 0)" - " t6 values('2020-01-01 00:01:02.000', 0)" - " t7 values('2020-01-01 00:01:02.000', 0)" - " t8 values('2020-01-01 00:01:02.000', 0)" - " t9 values('2020-01-01 00:01:02.000', 0)"); - int affected = taos_affected_rows(result); - if (affected != 18) { - printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); - } - taos_free_result(result); - // super tables subscription - usleep(1000000); -} - -static int print_result(TAOS_RES* res, int blockFetch) { - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(res); - TAOS_FIELD* fields = taos_fetch_fields(res); - int nRows = 0; - - if (blockFetch) { - int rows = 0; - while ((rows = taos_fetch_block(res, &row))) { - // for (int i = 0; i < rows; i++) { - // char temp[256]; - // taos_print_row(temp, row + i, fields, num_fields); - // puts(temp); - // } - nRows += rows; - } - } else { - while ((row = taos_fetch_row(res))) { - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - puts(temp); - nRows++; - } - } - - printf("%d rows consumed.\n", nRows); - return nRows; -} - -static void check_row_count(int line, TAOS_RES* res, int expected) { - int actual = print_result(res, expected % 2); - if (actual != expected) { - printf("\033[31mline %d: row count mismatch, expected: %d, actual: %d\033[0m\n", line, expected, actual); - } else { - printf("line %d: %d rows consumed as expected\n", line, actual); - } -} - -static void verify_query(TAOS* taos) { - prepare_data(taos); - - int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9"); - if (code != 0) { - printf("\033[31mfailed to load table info: 0x%08x\033[0m\n", code); - } - - code = taos_validate_sql(taos, "select * from nonexisttable"); - if (code == 0) { - printf("\033[31mimpossible, the table does not exists\033[0m\n"); - } - - code = taos_validate_sql(taos, "select * from meters"); - if (code != 0) { - printf("\033[31mimpossible, the table does exists: 0x%08x\033[0m\n", code); - } - - TAOS_RES* res = taos_query_with_reqid(taos, "select * from meters", genReqid()); - check_row_count(__LINE__, res, 18); - printf("result precision is: %d\n", taos_result_precision(res)); - int c = taos_field_count(res); - printf("field count is: %d\n", c); - int* lengths = taos_fetch_lengths(res); - for (int i = 0; i < c; i++) { - printf("length of column %d is %d\n", i, lengths[i]); - } - taos_free_result(res); - - res = taos_query_with_reqid(taos, "select * from t0", genReqid()); - check_row_count(__LINE__, res, 3); - taos_free_result(res); - - res = taos_query_with_reqid(taos, "select * from nonexisttable", genReqid()); - code = taos_errno(res); - printf("code=%d, error msg=%s\n", code, taos_errstr(res)); - taos_free_result(res); - - res = taos_query_with_reqid(taos, "select * from meters", genReqid()); - taos_stop_query(res); - taos_free_result(res); -} - -void subscribe_callback(TAOS_SUB* tsub, TAOS_RES* res, void* param, int code) { - int rows = print_result(res, *(int*)param); - printf("%d rows consumed in subscribe_callback\n", rows); -} - -static void verify_subscribe(TAOS* taos) { - prepare_data(taos); - - TAOS_SUB* tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); - TAOS_RES* res = taos_consume(tsub); - check_row_count(__LINE__, res, 18); - - res = taos_consume(tsub); - check_row_count(__LINE__, res, 0); - - TAOS_RES* result; - result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); - taos_free_result(result); - result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); - taos_free_result(result); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 2); - - result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); - taos_free_result(result); - result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); - taos_free_result(result); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 2); - - result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); - taos_free_result(result); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 1); - - // keep progress information and restart subscription - taos_unsubscribe(tsub, 1); - result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); - taos_free_result(result); - tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 24); - - // keep progress information and continue previous subscription - taos_unsubscribe(tsub, 1); - tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 0); - - // don't keep progress information and continue previous subscription - taos_unsubscribe(tsub, 0); - tsub = taos_subscribe(taos, 0, "test", "select * from meters;", NULL, NULL, 0); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 24); - - // single meter subscription - - taos_unsubscribe(tsub, 0); - tsub = taos_subscribe(taos, 0, "test", "select * from t0;", NULL, NULL, 0); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 5); - - res = taos_consume(tsub); - check_row_count(__LINE__, res, 0); - - result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); - taos_free_result(result); - res = taos_consume(tsub); - check_row_count(__LINE__, res, 1); - - taos_unsubscribe(tsub, 0); - - int blockFetch = 0; - tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); - usleep(2000000); - result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); - taos_free_result(result); - usleep(2000000); - taos_unsubscribe(tsub, 0); -} - -void verify_prepare(TAOS* taos) { - TAOS_RES* result = taos_query(taos, "drop database if exists test;"); - taos_free_result(result); - - usleep(100000); - result = taos_query(taos, "create database test;"); - - int code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - - taos_free_result(result); - - usleep(100000); - taos_select_db(taos, "test"); - - // create table - const char* sql = - "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " - "binary(40), blob nchar(10))"; - result = taos_query_with_reqid(taos, sql, genReqid()); - code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); - - // insert 10 records - struct { - int64_t ts; - int8_t b; - int8_t v1; - int16_t v2; - int32_t v4; - int64_t v8; - float f4; - double f8; - char bin[40]; - char blob[80]; - } v = {0}; - - TAOS_STMT* stmt = taos_stmt_init(taos); - TAOS_BIND params[10]; - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts); - params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; - params[0].is_null = NULL; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b); - params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; - params[1].is_null = NULL; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1); - params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; - params[2].is_null = NULL; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2); - params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; - params[3].is_null = NULL; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4); - params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; - params[4].is_null = NULL; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8); - params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; - params[5].is_null = NULL; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4); - params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; - params[6].is_null = NULL; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8); - params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; - params[7].is_null = NULL; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin); - params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; - params[8].is_null = NULL; - - strcpy(v.blob, "一二三四五六七八九十"); - params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[9].buffer_length = strlen(v.blob); - params[9].buffer = v.blob; - params[9].length = ¶ms[9].buffer_length; - params[9].is_null = NULL; - - int is_null = 1; - - sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; - code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0) { - printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - v.ts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts += 1; - for (int j = 1; j < 10; ++j) { - params[j].is_null = ((i == j) ? &is_null : 0); - } - v.b = (int8_t)i % 2; - v.v1 = (int8_t)i; - v.v2 = (int16_t)(i * 2); - v.v4 = (int32_t)(i * 4); - v.v8 = (int64_t)(i * 8); - v.f4 = (float)(i * 40); - v.f8 = (double)(i * 80); - for (int j = 0; j < sizeof(v.bin); ++j) { - v.bin[j] = (char)(i + '0'); - } - - taos_stmt_bind_param(stmt, params); - taos_stmt_add_batch(stmt); - } - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - taos_stmt_close(stmt); - - // query the records - stmt = taos_stmt_init(taos); - taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); - v.v1 = 5; - v.v2 = 15; - taos_stmt_bind_param(stmt, params + 2); - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - result = taos_stmt_use_result(stmt); - - TAOS_ROW row; - int rows = 0; - int num_fields = taos_num_fields(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - - // fetch the records row by row - while ((row = taos_fetch_row(result))) { - char temp[256] = {0}; - rows++; - taos_print_row(temp, row, fields, num_fields); - printf("%s\n", temp); - } - - taos_free_result(result); - taos_stmt_close(stmt); -} - -void verify_prepare2(TAOS* taos) { - TAOS_RES* result = taos_query(taos, "drop database if exists test;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database test;"); - - int code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); - - usleep(100000); - taos_select_db(taos, "test"); - - // create table - const char* sql = - "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " - "binary(40), blob nchar(10))"; - result = taos_query(taos, sql); - code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); - - // insert 10 records - struct { - int64_t ts[10]; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - char blob[10][80]; - } v; - - int32_t* t8_len = malloc(sizeof(int32_t) * 10); - int32_t* t16_len = malloc(sizeof(int32_t) * 10); - int32_t* t32_len = malloc(sizeof(int32_t) * 10); - int32_t* t64_len = malloc(sizeof(int32_t) * 10); - int32_t* float_len = malloc(sizeof(int32_t) * 10); - int32_t* double_len = malloc(sizeof(int32_t) * 10); - int32_t* bin_len = malloc(sizeof(int32_t) * 10); - int32_t* blob_len = malloc(sizeof(int32_t) * 10); - - TAOS_STMT* stmt = taos_stmt_init(taos); - TAOS_MULTI_BIND params[10]; - char is_null[10] = {0}; - - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts[0]); - params[0].buffer = v.ts; - params[0].length = t64_len; - params[0].is_null = is_null; - params[0].num = 10; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b[0]); - params[1].buffer = v.b; - params[1].length = t8_len; - params[1].is_null = is_null; - params[1].num = 10; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1[0]); - params[2].buffer = v.v1; - params[2].length = t8_len; - params[2].is_null = is_null; - params[2].num = 10; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2[0]); - params[3].buffer = v.v2; - params[3].length = t16_len; - params[3].is_null = is_null; - params[3].num = 10; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4[0]); - params[4].buffer = v.v4; - params[4].length = t32_len; - params[4].is_null = is_null; - params[4].num = 10; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8[0]); - params[5].buffer = v.v8; - params[5].length = t64_len; - params[5].is_null = is_null; - params[5].num = 10; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4[0]); - params[6].buffer = v.f4; - params[6].length = float_len; - params[6].is_null = is_null; - params[6].num = 10; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8[0]); - params[7].buffer = v.f8; - params[7].length = double_len; - params[7].is_null = is_null; - params[7].num = 10; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin[0]); - params[8].buffer = v.bin; - params[8].length = bin_len; - params[8].is_null = is_null; - params[8].num = 10; - - params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[9].buffer_length = sizeof(v.blob[0]); - params[9].buffer = v.blob; - params[9].length = blob_len; - params[9].is_null = is_null; - params[9].num = 10; - - sql = "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)"; - code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0) { - printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - code = taos_stmt_set_tbname(stmt, "m1"); - if (code != 0) { - printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - int64_t ts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = ts++; - is_null[i] = 0; - - v.b[i] = (int8_t)i % 2; - v.v1[i] = (int8_t)i; - v.v2[i] = (int16_t)(i * 2); - v.v4[i] = (int32_t)(i * 4); - v.v8[i] = (int64_t)(i * 8); - v.f4[i] = (float)(i * 40); - v.f8[i] = (double)(i * 80); - for (int j = 0; j < sizeof(v.bin[0]); ++j) { - v.bin[i][j] = (char)(i + '0'); - } - strcpy(v.blob[i], "一二三四五六七八九十"); - - t8_len[i] = sizeof(int8_t); - t16_len[i] = sizeof(int16_t); - t32_len[i] = sizeof(int32_t); - t64_len[i] = sizeof(int64_t); - float_len[i] = sizeof(float); - double_len[i] = sizeof(double); - bin_len[i] = sizeof(v.bin[0]); - blob_len[i] = (int32_t)strlen(v.blob[i]); - } - - taos_stmt_bind_param_batch(stmt, params); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - taos_stmt_close(stmt); - - // query the records - stmt = taos_stmt_init(taos); - taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); - TAOS_BIND qparams[2]; - - int8_t v1 = 5; - int16_t v2 = 15; - qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; - qparams[0].buffer_length = sizeof(v1); - qparams[0].buffer = &v1; - qparams[0].length = &qparams[0].buffer_length; - qparams[0].is_null = NULL; - - qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - qparams[1].buffer_length = sizeof(v2); - qparams[1].buffer = &v2; - qparams[1].length = &qparams[1].buffer_length; - qparams[1].is_null = NULL; - - taos_stmt_bind_param(stmt, qparams); - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - result = taos_stmt_use_result(stmt); - - TAOS_ROW row; - int rows = 0; - int num_fields = taos_num_fields(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - - // fetch the records row by row - while ((row = taos_fetch_row(result))) { - char temp[256] = {0}; - rows++; - taos_print_row(temp, row, fields, num_fields); - printf("%s\n", temp); - } - - taos_free_result(result); - taos_stmt_close(stmt); - - free(t8_len); - free(t16_len); - free(t32_len); - free(t64_len); - free(float_len); - free(double_len); - free(bin_len); - free(blob_len); -} - -void verify_prepare3(TAOS* taos) { - TAOS_RES* result = taos_query(taos, "drop database if exists test;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database test;"); - - int code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); - - usleep(100000); - taos_select_db(taos, "test"); - - // create table - const char* sql = - "create stable st1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " - "binary(40), blob nchar(10)) tags (id1 int, id2 binary(40))"; - result = taos_query(taos, sql); - code = taos_errno(result); - if (code != 0) { - printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); - taos_free_result(result); - return; - } - taos_free_result(result); - - TAOS_BIND tags[2]; - - int32_t id1 = 1; - char id2[40] = "abcdefghijklmnopqrstuvwxyz0123456789"; - uintptr_t id2_len = strlen(id2); - - tags[0].buffer_type = TSDB_DATA_TYPE_INT; - tags[0].buffer_length = sizeof(int); - tags[0].buffer = &id1; - tags[0].length = NULL; - tags[0].is_null = NULL; - - tags[1].buffer_type = TSDB_DATA_TYPE_BINARY; - tags[1].buffer_length = sizeof(id2); - tags[1].buffer = id2; - tags[1].length = &id2_len; - tags[1].is_null = NULL; - - // insert 10 records - struct { - int64_t ts[10]; - int8_t b[10]; - int8_t v1[10]; - int16_t v2[10]; - int32_t v4[10]; - int64_t v8[10]; - float f4[10]; - double f8[10]; - char bin[10][40]; - char blob[10][80]; - } v; - - int32_t* t8_len = malloc(sizeof(int32_t) * 10); - int32_t* t16_len = malloc(sizeof(int32_t) * 10); - int32_t* t32_len = malloc(sizeof(int32_t) * 10); - int32_t* t64_len = malloc(sizeof(int32_t) * 10); - int32_t* float_len = malloc(sizeof(int32_t) * 10); - int32_t* double_len = malloc(sizeof(int32_t) * 10); - int32_t* bin_len = malloc(sizeof(int32_t) * 10); - int32_t* blob_len = malloc(sizeof(int32_t) * 10); - - TAOS_STMT* stmt = taos_stmt_init(taos); - TAOS_MULTI_BIND params[10]; - char is_null[10] = {0}; - - params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; - params[0].buffer_length = sizeof(v.ts[0]); - params[0].buffer = v.ts; - params[0].length = t64_len; - params[0].is_null = is_null; - params[0].num = 10; - - params[1].buffer_type = TSDB_DATA_TYPE_BOOL; - params[1].buffer_length = sizeof(v.b[0]); - params[1].buffer = v.b; - params[1].length = t8_len; - params[1].is_null = is_null; - params[1].num = 10; - - params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; - params[2].buffer_length = sizeof(v.v1[0]); - params[2].buffer = v.v1; - params[2].length = t8_len; - params[2].is_null = is_null; - params[2].num = 10; - - params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; - params[3].buffer_length = sizeof(v.v2[0]); - params[3].buffer = v.v2; - params[3].length = t16_len; - params[3].is_null = is_null; - params[3].num = 10; - - params[4].buffer_type = TSDB_DATA_TYPE_INT; - params[4].buffer_length = sizeof(v.v4[0]); - params[4].buffer = v.v4; - params[4].length = t32_len; - params[4].is_null = is_null; - params[4].num = 10; - - params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; - params[5].buffer_length = sizeof(v.v8[0]); - params[5].buffer = v.v8; - params[5].length = t64_len; - params[5].is_null = is_null; - params[5].num = 10; - - params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; - params[6].buffer_length = sizeof(v.f4[0]); - params[6].buffer = v.f4; - params[6].length = float_len; - params[6].is_null = is_null; - params[6].num = 10; - - params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; - params[7].buffer_length = sizeof(v.f8[0]); - params[7].buffer = v.f8; - params[7].length = double_len; - params[7].is_null = is_null; - params[7].num = 10; - - params[8].buffer_type = TSDB_DATA_TYPE_BINARY; - params[8].buffer_length = sizeof(v.bin[0]); - params[8].buffer = v.bin; - params[8].length = bin_len; - params[8].is_null = is_null; - params[8].num = 10; - - params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[9].buffer_length = sizeof(v.blob[0]); - params[9].buffer = v.blob; - params[9].length = blob_len; - params[9].is_null = is_null; - params[9].num = 10; - - sql = "insert into ? using st1 tags(?,?) values(?,?,?,?,?,?,?,?,?,?)"; - code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0) { - printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - code = taos_stmt_set_tbname_tags(stmt, "m1", tags); - if (code != 0) { - printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - int64_t ts = 1591060628000; - for (int i = 0; i < 10; ++i) { - v.ts[i] = ts++; - is_null[i] = 0; - - v.b[i] = (int8_t)i % 2; - v.v1[i] = (int8_t)i; - v.v2[i] = (int16_t)(i * 2); - v.v4[i] = (int32_t)(i * 4); - v.v8[i] = (int64_t)(i * 8); - v.f4[i] = (float)(i * 40); - v.f8[i] = (double)(i * 80); - for (int j = 0; j < sizeof(v.bin[0]); ++j) { - v.bin[i][j] = (char)(i + '0'); - } - strcpy(v.blob[i], "一二三四五六七八九十"); - - t8_len[i] = sizeof(int8_t); - t16_len[i] = sizeof(int16_t); - t32_len[i] = sizeof(int32_t); - t64_len[i] = sizeof(int64_t); - float_len[i] = sizeof(float); - double_len[i] = sizeof(double); - bin_len[i] = sizeof(v.bin[0]); - blob_len[i] = (int32_t)strlen(v.blob[i]); - } - - taos_stmt_bind_param_batch(stmt, params); - taos_stmt_add_batch(stmt); - - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - taos_stmt_close(stmt); - - // query the records - stmt = taos_stmt_init(taos); - taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); - - TAOS_BIND qparams[2]; - - int8_t v1 = 5; - int16_t v2 = 15; - qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; - qparams[0].buffer_length = sizeof(v1); - qparams[0].buffer = &v1; - qparams[0].length = &qparams[0].buffer_length; - qparams[0].is_null = NULL; - - qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; - qparams[1].buffer_length = sizeof(v2); - qparams[1].buffer = &v2; - qparams[1].length = &qparams[1].buffer_length; - qparams[1].is_null = NULL; - - taos_stmt_bind_param(stmt, qparams); - if (taos_stmt_execute(stmt) != 0) { - printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - return; - } - - result = taos_stmt_use_result(stmt); - - TAOS_ROW row; - int rows = 0; - int num_fields = taos_num_fields(result); - TAOS_FIELD* fields = taos_fetch_fields(result); - - // fetch the records row by row - while ((row = taos_fetch_row(result))) { - char temp[256] = {0}; - rows++; - taos_print_row(temp, row, fields, num_fields); - printf("%s\n", temp); - } - - taos_free_result(result); - taos_stmt_close(stmt); - - free(t8_len); - free(t16_len); - free(t32_len); - free(t64_len); - free(float_len); - free(double_len); - free(bin_len); - free(blob_len); -} - -void retrieve_callback(void* param, TAOS_RES* tres, int numOfRows) { - if (numOfRows > 0) { - printf("%d rows async retrieved\n", numOfRows); - taos_fetch_rows_a(tres, retrieve_callback, param); - } else { - if (numOfRows < 0) { - printf("\033[31masync retrieve failed, code: %d\033[0m\n", numOfRows); - } else { - printf("async retrieve completed\n"); - } - taos_free_result(tres); - } -} - -void select_callback(void* param, TAOS_RES* tres, int code) { - if (code == 0 && tres) { - taos_fetch_rows_a(tres, retrieve_callback, param); - } else { - printf("\033[31masync select failed, code: %d\033[0m\n", code); - } -} - -void verify_async(TAOS* taos) { - prepare_data(taos); - taos_query_a(taos, "select * from meters", select_callback, NULL); - usleep(1000000); -} - -void stream_callback(void* param, TAOS_RES* res, TAOS_ROW row) { - if (res == NULL || row == NULL) { - return; - } - - int num_fields = taos_num_fields(res); - TAOS_FIELD* fields = taos_fetch_fields(res); - - printf("got one row from stream_callback\n"); - char temp[256] = {0}; - taos_print_row(temp, row, fields, num_fields); - puts(temp); -} - -void verify_stream(TAOS* taos) { - prepare_data(taos); - TAOS_STREAM* strm = - taos_open_stream(taos, "select count(*) from meters interval(1m)", stream_callback, 0, NULL, NULL); - printf("waiting for stream data\n"); - usleep(100000); - TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); - taos_free_result(result); - usleep(200000000); - taos_close_stream(strm); -} - -int32_t verify_schema_less(TAOS* taos) { - TAOS_RES* result; - result = taos_query(taos, "drop database if exists test;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database test precision 'us' update 1;"); - taos_free_result(result); - usleep(100000); - - taos_select_db(taos, "test"); - result = taos_query(taos, "create stable ste(ts timestamp, f int) tags(t1 bigint)"); - taos_free_result(result); - usleep(100000); - - int code = 0; - - char* lines[] = { - "st,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", - "st,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns", - "ste,t2=5f64,t3=L\"ste\" c1=true,c2=4i64,c3=\"iam\" 1626056811823316532ns", - "st,t1=4i64,t2=5f64,t3=\"t4\" c1=3i64,c3=L\"passitagain\",c2=true,c4=5f64 1626006833642000000ns", - "ste,t2=5f64,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532ns", - "ste,t2=5f64,t3=L\"ste2\" c3=\"iamszhou\",c4=false,c5=32i8,c6=64i16,c7=32i32,c8=88.88f32 1626056812843316532ns", - "st,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " - "1626006933640000000ns", - "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64,c6=7u64 " - "1626006933640000000ns", - "stf,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 " - "1626006933641000000ns"}; - - code = taos_insert_lines(taos, lines, sizeof(lines) / sizeof(char*)); - - char* lines2[] = { - "stg,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", - "stg,t1=4i64,t3=\"t4\",t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin\",c2=true,c4=5f64,c5=5f64 1626006833640000000ns"}; - code = taos_insert_lines(taos, &lines2[0], 1); - code = taos_insert_lines(taos, &lines2[1], 1); - - char* lines3[] = { - "sth,t1=4i64,t2=5f64,t4=5f64,ID=\"childtable\" c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 " - "1626006933641ms", - "sth,t1=4i64,t2=5f64,t4=5f64 c1=3i64,c3=L\"passitagin_stf\",c2=false,c5=5f64,c6=7u64 1626006933654ms"}; - code = taos_insert_lines(taos, lines3, 2); - - char* lines4[] = {"st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", - "dgtyqodr,t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"}; - code = taos_insert_lines(taos, lines4, 2); - - char* lines5[] = { - "zqlbgs,id=\"zqlbgs_39302_21680\",t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" - "\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000ns", - "zqlbgs,t9=f,id=\"zqlbgs_39302_21680\",t0=f,t1=127i8,t11=127i8,t2=32767i16,t3=2147483647i32,t4=" - "9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t10=" - "L\"ncharTagValue\" " - "c10=f,c0=f,c1=127i8,c12=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\" 1626006833639000000ns"}; - code = taos_insert_lines(taos, &lines5[0], 1); - code = taos_insert_lines(taos, &lines5[1], 1); - - char* lines6[] = {"st123456,t1=3i64,t2=4f64,t3=\"t3\" c1=3i64,c3=L\"passit\",c2=false,c4=4f64 1626006833639000000ns", - "dgtyqodr,t2=5f64,t3=L\"ste\" c1=tRue,c2=4i64,c3=\"iam\" 1626056811823316532ns"}; - code = taos_insert_lines(taos, lines6, 2); - return (code); -} - -void verify_telnet_insert(TAOS* taos) { - TAOS_RES* result; - - result = taos_query(taos, "drop database if exists db;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database db precision 'ms';"); - taos_free_result(result); - usleep(100000); - - (void)taos_select_db(taos, "db"); - int32_t code = 0; - - /* metric */ - char* lines0[] = { - "stb0_0 1626006833639000000ns 4i8 host=\"host0\",interface=\"eth0\"", - "stb0_1 1626006833639000000ns 4i8 host=\"host0\",interface=\"eth0\"", - "stb0_2 1626006833639000000ns 4i8 host=\"host0\",interface=\"eth0\"", - }; - code = taos_insert_telnet_lines(taos, lines0, 3); - if (code) { - printf("lines0 code: %d, %s.\n", code, tstrerror(code)); - } - - /* timestamp */ - char* lines1[] = { - "stb1 1626006833s 1i8 host=\"host0\"", "stb1 1626006833639000000ns 2i8 host=\"host0\"", - "stb1 1626006833640000us 3i8 host=\"host0\"", "stb1 1626006833641123 4i8 host=\"host0\"", - "stb1 1626006833651ms 5i8 host=\"host0\"", "stb1 0 6i8 host=\"host0\"", - }; - code = taos_insert_telnet_lines(taos, lines1, 6); - if (code) { - printf("lines1 code: %d, %s.\n", code, tstrerror(code)); - } - - /* metric value */ - // tinyin - char* lines2_0[] = {"stb2_0 1626006833651ms -127i8 host=\"host0\"", "stb2_0 1626006833652ms 127i8 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_0, 2); - if (code) { - printf("lines2_0 code: %d, %s.\n", code, tstrerror(code)); - } - - // smallint - char* lines2_1[] = {"stb2_1 1626006833651ms -32767i16 host=\"host0\"", - "stb2_1 1626006833652ms 32767i16 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_1, 2); - if (code) { - printf("lines2_1 code: %d, %s.\n", code, tstrerror(code)); - } - - // int - char* lines2_2[] = {"stb2_2 1626006833651ms -2147483647i32 host=\"host0\"", - "stb2_2 1626006833652ms 2147483647i32 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_2, 2); - if (code) { - printf("lines2_2 code: %d, %s.\n", code, tstrerror(code)); - } - - // bigint - char* lines2_3[] = {"stb2_3 1626006833651ms -9223372036854775807i64 host=\"host0\"", - "stb2_3 1626006833652ms 9223372036854775807i64 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_3, 2); - if (code) { - printf("lines2_3 code: %d, %s.\n", code, tstrerror(code)); - } - - // float - char* lines2_4[] = { - "stb2_4 1626006833610ms 3f32 host=\"host0\"", "stb2_4 1626006833620ms -3f32 host=\"host0\"", - "stb2_4 1626006833630ms 3.4f32 host=\"host0\"", "stb2_4 1626006833640ms -3.4f32 host=\"host0\"", - "stb2_4 1626006833650ms 3.4E10f32 host=\"host0\"", "stb2_4 1626006833660ms -3.4e10f32 host=\"host0\"", - "stb2_4 1626006833670ms 3.4E+2f32 host=\"host0\"", "stb2_4 1626006833680ms -3.4e-2f32 host=\"host0\"", - "stb2_4 1626006833690ms 3.15 host=\"host0\"", "stb2_4 1626006833700ms 3.4E38f32 host=\"host0\"", - "stb2_4 1626006833710ms -3.4E38f32 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_4, 11); - if (code) { - printf("lines2_4 code: %d, %s.\n", code, tstrerror(code)); - } - - // double - char* lines2_5[] = { - "stb2_5 1626006833610ms 3f64 host=\"host0\"", "stb2_5 1626006833620ms -3f64 host=\"host0\"", - "stb2_5 1626006833630ms 3.4f64 host=\"host0\"", "stb2_5 1626006833640ms -3.4f64 host=\"host0\"", - "stb2_5 1626006833650ms 3.4E10f64 host=\"host0\"", "stb2_5 1626006833660ms -3.4e10f64 host=\"host0\"", - "stb2_5 1626006833670ms 3.4E+2f64 host=\"host0\"", "stb2_5 1626006833680ms -3.4e-2f64 host=\"host0\"", - "stb2_5 1626006833690ms 1.7E308f64 host=\"host0\"", "stb2_5 1626006833700ms -1.7E308f64 host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_5, 10); - if (code) { - printf("lines2_5 code: %d, %s.\n", code, tstrerror(code)); - } - - // bool - char* lines2_6[] = {"stb2_6 1626006833610ms t host=\"host0\"", "stb2_6 1626006833620ms T host=\"host0\"", - "stb2_6 1626006833630ms true host=\"host0\"", "stb2_6 1626006833640ms True host=\"host0\"", - "stb2_6 1626006833650ms TRUE host=\"host0\"", "stb2_6 1626006833660ms f host=\"host0\"", - "stb2_6 1626006833670ms F host=\"host0\"", "stb2_6 1626006833680ms false host=\"host0\"", - "stb2_6 1626006833690ms False host=\"host0\"", "stb2_6 1626006833700ms FALSE host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_6, 10); - if (code) { - printf("lines2_6 code: %d, %s.\n", code, tstrerror(code)); - } - - // binary - char* lines2_7[] = {"stb2_7 1626006833610ms \"binary_val.!@#$%^&*\" host=\"host0\"", - "stb2_7 1626006833620ms \"binary_val.:;,./?|+-=\" host=\"host0\"", - "stb2_7 1626006833630ms \"binary_val.()[]{}<>\" host=\"host0\""}; - code = taos_insert_telnet_lines(taos, lines2_7, 3); - if (code) { - printf("lines2_7 code: %d, %s.\n", code, tstrerror(code)); - } - - // nchar - char* lines2_8[] = { - "stb2_8 1626006833610ms L\"nchar_val数值一\" host=\"host0\"", - "stb2_8 1626006833620ms L\"nchar_val数值二\" host=\"host0\"", - }; - code = taos_insert_telnet_lines(taos, lines2_8, 2); - if (code) { - printf("lines2_8 code: %d, %s.\n", code, tstrerror(code)); - } - - /* tags */ - // tag value types - char* lines3_0[] = { - "stb3_0 1626006833610ms 1 " - "t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=3.4E38f32,t6=1.7E308f64,t7=true,t8=\"binary_" - "val_1\",t9=L\"标签值1\"", - "stb3_0 1626006833610ms 2 " - "t1=-127i8,t2=-32767i16,t3=-2147483647i32,t4=-9223372036854775807i64,t5=-3.4E38f32,t6=-1.7E308f64,t7=false,t8=" - "\"binary_val_2\",t9=L\"标签值2\""}; - code = taos_insert_telnet_lines(taos, lines3_0, 2); - if (code) { - printf("lines3_0 code: %d, %s.\n", code, tstrerror(code)); - } - - // tag ID as child table name - char* lines3_1[] = {"stb3_1 1626006833610ms 1 id=\"child_table1\",host=\"host1\"", - "stb3_1 1626006833610ms 2 host=\"host2\",iD=\"child_table2\"", - "stb3_1 1626006833610ms 3 ID=\"child_table3\",host=\"host3\""}; - code = taos_insert_telnet_lines(taos, lines3_1, 3); - if (code) { - printf("lines3_1 code: %d, %s.\n", code, tstrerror(code)); - } - - return; -} - -void verify_json_insert(TAOS* taos) { - TAOS_RES* result; - - result = taos_query(taos, "drop database if exists db;"); - taos_free_result(result); - usleep(100000); - result = taos_query(taos, "create database db precision 'ms';"); - taos_free_result(result); - usleep(100000); - - (void)taos_select_db(taos, "db"); - int32_t code = 0; - - char* message = - "{ \ - \"metric\":\"cpu_load_0\", \ - \"timestamp\": 1626006833610123, \ - \"value\": 55.5, \ - \"tags\": \ - { \ - \"host\": \"ubuntu\", \ - \"interface1\": \"eth0\", \ - \"Id\": \"tb0\" \ - } \ - }"; - - code = taos_insert_json_payload(taos, message); - if (code) { - printf("payload_0 code: %d, %s.\n", code, tstrerror(code)); - } - - char* message1 = - "[ \ - { \ - \"metric\":\"cpu_load_1\", \ - \"timestamp\": 1626006833610123, \ - \"value\": 55.5, \ - \"tags\": \ - { \ - \"host\": \"ubuntu\", \ - \"interface\": \"eth1\", \ - \"Id\": \"tb1\" \ - } \ - }, \ - { \ - \"metric\":\"cpu_load_2\", \ - \"timestamp\": 1626006833610123, \ - \"value\": 55.5, \ - \"tags\": \ - { \ - \"host\": \"ubuntu\", \ - \"interface\": \"eth2\", \ - \"Id\": \"tb2\" \ - } \ - } \ - ]"; - - code = taos_insert_json_payload(taos, message1); - if (code) { - printf("payload_1 code: %d, %s.\n", code, tstrerror(code)); - } - - char* message2 = - "[ \ - { \ - \"metric\":\"cpu_load_3\", \ - \"timestamp\": \ - { \ - \"value\": 1626006833610123, \ - \"type\": \"us\" \ - }, \ - \"value\": \ - { \ - \"value\": 55, \ - \"type\": \"int\" \ - }, \ - \"tags\": \ - { \ - \"host\": \ - { \ - \"value\": \"ubuntu\", \ - \"type\": \"binary\" \ - }, \ - \"interface\": \ - { \ - \"value\": \"eth3\", \ - \"type\": \"nchar\" \ - }, \ - \"ID\": \"tb3\", \ - \"port\": \ - { \ - \"value\": 4040, \ - \"type\": \"int\" \ - } \ - } \ - }, \ - { \ - \"metric\":\"cpu_load_4\", \ - \"timestamp\": 1626006833610123, \ - \"value\": 66.6, \ - \"tags\": \ - { \ - \"host\": \"ubuntu\", \ - \"interface\": \"eth4\", \ - \"Id\": \"tb4\" \ - } \ - } \ - ]"; - code = taos_insert_json_payload(taos, message2); - if (code) { - printf("payload_2 code: %d, %s.\n", code, tstrerror(code)); - } - - cJSON *payload, *tags; - char* payload_str; - - /* Default format */ - // number - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_0"); - cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_0 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // true - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_1"); - cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); - cJSON_AddTrueToObject(payload, "value"); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_1 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // false - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_2"); - cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); - cJSON_AddFalseToObject(payload, "value"); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_2 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // string - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_3"); - cJSON_AddNumberToObject(payload, "timestamp", 1626006833610123); - cJSON_AddStringToObject(payload, "value", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_3 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // timestamp 0 -> current time - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_4"); - cJSON_AddNumberToObject(payload, "timestamp", 0); - cJSON_AddNumberToObject(payload, "value", 123); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_4 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // ID - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb0_5"); - cJSON_AddNumberToObject(payload, "timestamp", 0); - cJSON_AddNumberToObject(payload, "value", 123); - tags = cJSON_CreateObject(); - cJSON_AddStringToObject(tags, "ID", "tb0_5"); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddStringToObject(tags, "iD", "tb000"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddStringToObject(tags, "id", "tb555"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload0_5 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - /* Nested format */ - // timestamp - cJSON* timestamp; - // seconds - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb1_0"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload1_0 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // milleseconds - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb1_1"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833610); - cJSON_AddStringToObject(timestamp, "type", "ms"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload1_1 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // microseconds - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb1_2"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833610123); - cJSON_AddStringToObject(timestamp, "type", "us"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload1_2 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // nanoseconds - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb1_3"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833610123321); - cJSON_AddStringToObject(timestamp, "type", "ns"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload1_3 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // now - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb1_4"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 0); - cJSON_AddStringToObject(timestamp, "type", "ns"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - cJSON_AddNumberToObject(payload, "value", 10); - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload1_4 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // metric value - cJSON* metric_val; - // bool - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_0"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddTrueToObject(metric_val, "value"); - cJSON_AddStringToObject(metric_val, "type", "bool"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_0 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // tinyint - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_1"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 127); - cJSON_AddStringToObject(metric_val, "type", "tinyint"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_1 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // smallint - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_2"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 32767); - cJSON_AddStringToObject(metric_val, "type", "smallint"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_2 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // int - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_3"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 2147483647); - cJSON_AddStringToObject(metric_val, "type", "int"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_3 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // bigint - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_4"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 9223372036854775807); - cJSON_AddStringToObject(metric_val, "type", "bigint"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_4 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // float - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_5"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 11.12345); - cJSON_AddStringToObject(metric_val, "type", "float"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_5 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // double - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_6"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddNumberToObject(metric_val, "value", 22.123456789); - cJSON_AddStringToObject(metric_val, "type", "double"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_6 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // binary - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_7"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddStringToObject(metric_val, "value", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddStringToObject(metric_val, "type", "binary"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_7 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // nchar - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb2_8"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddStringToObject(metric_val, "value", "你好"); - cJSON_AddStringToObject(metric_val, "type", "nchar"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - cJSON_AddTrueToObject(tags, "t1"); - cJSON_AddFalseToObject(tags, "t2"); - cJSON_AddNumberToObject(tags, "t3", 10); - cJSON_AddStringToObject(tags, "t4", "123_abc_.!@#$%^&*:;,./?|+-=()[]{}<>"); - cJSON_AddItemToObject(payload, "tags", tags); - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload2_8 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); - - // tag value - cJSON* tag; - - payload = cJSON_CreateObject(); - cJSON_AddStringToObject(payload, "metric", "stb3_0"); - - timestamp = cJSON_CreateObject(); - cJSON_AddNumberToObject(timestamp, "value", 1626006833); - cJSON_AddStringToObject(timestamp, "type", "s"); - cJSON_AddItemToObject(payload, "timestamp", timestamp); - - metric_val = cJSON_CreateObject(); - cJSON_AddStringToObject(metric_val, "value", "hello"); - cJSON_AddStringToObject(metric_val, "type", "nchar"); - cJSON_AddItemToObject(payload, "value", metric_val); - - tags = cJSON_CreateObject(); - - tag = cJSON_CreateObject(); - cJSON_AddTrueToObject(tag, "value"); - cJSON_AddStringToObject(tag, "type", "bool"); - cJSON_AddItemToObject(tags, "t1", tag); - - tag = cJSON_CreateObject(); - cJSON_AddFalseToObject(tag, "value"); - cJSON_AddStringToObject(tag, "type", "bool"); - cJSON_AddItemToObject(tags, "t2", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 127); - cJSON_AddStringToObject(tag, "type", "tinyint"); - cJSON_AddItemToObject(tags, "t3", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 32767); - cJSON_AddStringToObject(tag, "type", "smallint"); - cJSON_AddItemToObject(tags, "t4", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 2147483647); - cJSON_AddStringToObject(tag, "type", "int"); - cJSON_AddItemToObject(tags, "t5", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 9223372036854775807); - cJSON_AddStringToObject(tag, "type", "bigint"); - cJSON_AddItemToObject(tags, "t6", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 11.12345); - cJSON_AddStringToObject(tag, "type", "float"); - cJSON_AddItemToObject(tags, "t7", tag); - - tag = cJSON_CreateObject(); - cJSON_AddNumberToObject(tag, "value", 22.1234567890); - cJSON_AddStringToObject(tag, "type", "double"); - cJSON_AddItemToObject(tags, "t8", tag); - - tag = cJSON_CreateObject(); - cJSON_AddStringToObject(tag, "value", "binary_val"); - cJSON_AddStringToObject(tag, "type", "binary"); - cJSON_AddItemToObject(tags, "t9", tag); - - tag = cJSON_CreateObject(); - cJSON_AddStringToObject(tag, "value", "你好"); - cJSON_AddStringToObject(tag, "type", "nchar"); - cJSON_AddItemToObject(tags, "t10", tag); - - cJSON_AddItemToObject(payload, "tags", tags); - - payload_str = cJSON_Print(payload); - // printf("%s\n", payload_str); - - code = taos_insert_json_payload(taos, payload_str); - if (code) { - printf("payload3_0 code: %d, %s.\n", code, tstrerror(code)); - } - free(payload_str); - cJSON_Delete(payload); -} - -int main(int argc, char* argv[]) { - const char* host = "127.0.0.1"; - const char* user = "root"; - const char* passwd = "taosdata"; - - taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - TAOS* taos = taos_connect(host, user, passwd, "", 0); - if (taos == NULL) { - printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); - exit(1); - } - - char* info = taos_get_server_info(taos); - printf("server info: %s\n", info); - info = taos_get_client_info(taos); - printf("client info: %s\n", info); - - printf("************ verify schema-less *************\n"); - verify_schema_less(taos); - - printf("************ verify telnet-insert *************\n"); - verify_telnet_insert(taos); - - printf("************ verify json-insert *************\n"); - verify_json_insert(taos); - - printf("************ verify query *************\n"); - verify_query(taos); - - printf("********* verify async query **********\n"); - verify_async(taos); - - printf("*********** verify subscribe ************\n"); - verify_subscribe(taos); - - printf("************ verify prepare *************\n"); - verify_prepare(taos); - - printf("************ verify prepare2 *************\n"); - verify_prepare2(taos); - printf("************ verify prepare3 *************\n"); - verify_prepare3(taos); - - printf("************ verify stream *************\n"); - verify_stream(taos); - printf("done\n"); - taos_close(taos); - taos_cleanup(); -} diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index 07e61b871ed2bd8e857861d169973f2ff6153a72..83d217769b603ee54f2d5250bdf412990de4b76a 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -23,8 +23,8 @@ #include #include #include - -#include "../../../include/client/taos.h" +#include +#include "taos.h" int points = 5; int numOfTables = 3; @@ -230,7 +230,7 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code) if (tablesInsertProcessed >= numOfTables) { gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - printf("%lld mseconds to insert %d data points\n", (et - st) / 1000, points*numOfTables); + printf("%" PRId64 " mseconds to insert %d data points\n", (et - st) / 1000, points*numOfTables); } } @@ -267,7 +267,7 @@ void taos_retrieve_call_back(void *param, TAOS_RES *tres, int numOfRows) if (tablesSelectProcessed >= numOfTables) { gettimeofday(&systemTime, NULL); et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - printf("%lld mseconds to query %d data rows\n", (et - st) / 1000, points * numOfTables); + printf("%" PRId64 " mseconds to query %d data rows\n", (et - st) / 1000, points * numOfTables); } taos_free_result(tres); diff --git a/examples/c/demo.c b/examples/c/demo.c index c3d9a5e96a9dbd3cc2f33e047f3befabfcaf9ecb..2eda8cd811804606eeb4d04be11350cfd0c7a1f8 100644 --- a/examples/c/demo.c +++ b/examples/c/demo.c @@ -20,7 +20,7 @@ #include #include #include -#include "../../../include/client/taos.h" // TAOS header file +#include "taos.h" // TAOS header file static void queryDB(TAOS *taos, char *command) { int i; diff --git a/examples/c/epoll.c b/examples/c/epoll.c deleted file mode 100644 index 284268ac4328b5bc814ab8d30931ec92c5c11523..0000000000000000000000000000000000000000 --- a/examples/c/epoll.c +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// how to use to do a pressure-test upon eok -// tester: cat /dev/urandom | nc -c -// testee: ./debug/build/bin/epoll -l > /dev/null -// compare against: nc -l > /dev/null -// monitor and compare : glances - -#ifdef __APPLE__ -#include "osEok.h" -#else // __APPLE__ -#include -#endif // __APPLE__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define D(fmt, ...) fprintf(stderr, "%s[%d]%s(): " fmt "\n", basename(__FILE__), __LINE__, __func__, ##__VA_ARGS__) -#define A(statement, fmt, ...) do { \ - if (statement) break; \ - fprintf(stderr, "%s[%d]%s(): assert [%s] failed: %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - #statement, errno, strerror(errno), \ - ##__VA_ARGS__); \ - abort(); \ -} while (0) - -#define E(fmt, ...) do { \ - fprintf(stderr, "%s[%d]%s(): %d[%s]: " fmt "\n", \ - basename(__FILE__), __LINE__, __func__, \ - errno, strerror(errno), \ - ##__VA_ARGS__); \ -} while (0) - -#include "os.h" - -typedef struct ep_s ep_t; -struct ep_s { - int ep; - - pthread_mutex_t lock; - int sv[2]; // 0 for read, 1 for write; - pthread_t thread; - - volatile unsigned int stopping:1; - volatile unsigned int waiting:1; - volatile unsigned int wakenup:1; -}; - -static int ep_dummy = 0; - -static ep_t* ep_create(void); -static void ep_destroy(ep_t *ep); -static void* routine(void* arg); -static int open_listen(unsigned short port); - -typedef struct fde_s fde_t; -struct fde_s { - int skt; - void (*on_event)(ep_t *ep, struct epoll_event *events, fde_t *client); -}; - -static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client); -static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client); - -#define usage(arg0, fmt, ...) do { \ - if (fmt[0]) { \ - fprintf(stderr, "" fmt "\n", ##__VA_ARGS__); \ - } \ - fprintf(stderr, "usage:\n"); \ - fprintf(stderr, " %s -l : specify listenning port\n", arg0); \ -} while (0) - -int main(int argc, char *argv[]) { - char *prg = basename(argv[0]); - if (argc==1) { - usage(prg, ""); - return 0; - } - ep_t* ep = ep_create(); - A(ep, "failed"); - for (int i=1; i=argc) { - usage(prg, "expecting after -l, but got nothing"); - return 1; // confirmed potential leakage - } - arg = argv[i]; - int port = atoi(arg); - int skt = open_listen(port); - if (skt==-1) continue; - fde_t *client = (fde_t*)calloc(1, sizeof(*client)); - if (!client) { - E("out of memory"); - close(skt); - continue; - } - client->skt = skt; - client->on_event = listen_event; - struct epoll_event ev = {0}; - ev.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ev.data.ptr = client; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ev), ""); - continue; - } - usage(prg, "unknown argument: [%s]", arg); - return 1; - } - char *line = NULL; - size_t linecap = 0; - ssize_t linelen; - while ((linelen = getline(&line, &linecap, stdin)) > 0) { - line[strlen(line)-1] = '\0'; - if (0==strcmp(line, "exit")) break; - if (0==strcmp(line, "quit")) break; - if (line==strstr(line, "close")) { - int fd = 0; - sscanf(line, "close %d", &fd); - if (fd<=2) { - fprintf(stderr, "fd [%d] invalid\n", fd); - continue; - } - A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, fd, NULL), ""); - continue; - } - if (strlen(line)==0) continue; - fprintf(stderr, "unknown cmd:[%s]\n", line); - } - ep_destroy(ep); - D(""); - return 0; -} - -ep_t* ep_create(void) { - ep_t *ep = (ep_t*)calloc(1, sizeof(*ep)); - A(ep, "out of memory"); - A(-1!=(ep->ep = epoll_create(1)), ""); - ep->sv[0] = -1; - ep->sv[1] = -1; - A(0==socketpair(AF_LOCAL, SOCK_STREAM, 0, ep->sv), ""); - A(0==pthread_mutex_init(&ep->lock, NULL), ""); - A(0==pthread_mutex_lock(&ep->lock), ""); - struct epoll_event ev = {0}; - ev.events = EPOLLIN; - ev.data.ptr = &ep_dummy; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, ep->sv[0], &ev), ""); - A(0==pthread_create(&ep->thread, NULL, routine, ep), ""); - A(0==pthread_mutex_unlock(&ep->lock), ""); - return ep; -} - -static void ep_destroy(ep_t *ep) { - A(ep, "invalid argument"); - ep->stopping = 1; - A(1==send(ep->sv[1], "1", 1, 0), ""); - A(0==pthread_join(ep->thread, NULL), ""); - A(0==pthread_mutex_destroy(&ep->lock), ""); - A(0==close(ep->sv[0]), ""); - A(0==close(ep->sv[1]), ""); - A(0==close(ep->ep), ""); - free(ep); -} - -static void* routine(void* arg) { - A(arg, "invalid argument"); - ep_t *ep = (ep_t*)arg; - - while (!ep->stopping) { - struct epoll_event evs[10]; - memset(evs, 0, sizeof(evs)); - - A(0==pthread_mutex_lock(&ep->lock), ""); - A(ep->waiting==0, "internal logic error"); - ep->waiting = 1; - A(0==pthread_mutex_unlock(&ep->lock), ""); - - int r = epoll_wait(ep->ep, evs, sizeof(evs)/sizeof(evs[0]), -1); - A(r>0, "indefinite epoll_wait shall not timeout:[%d]", r); - - A(0==pthread_mutex_lock(&ep->lock), ""); - A(ep->waiting==1, "internal logic error"); - ep->waiting = 0; - A(0==pthread_mutex_unlock(&ep->lock), ""); - - for (int i=0; idata.ptr == &ep_dummy) { - char c = '\0'; - A(1==recv(ep->sv[0], &c, 1, 0), "internal logic error"); - A(0==pthread_mutex_lock(&ep->lock), ""); - ep->wakenup = 0; - A(0==pthread_mutex_unlock(&ep->lock), ""); - continue; - } - A(ev->data.ptr, "internal logic error"); - fde_t *client = (fde_t*)ev->data.ptr; - client->on_event(ep, ev, client); - continue; - } - } - return NULL; -} - -static int open_listen(unsigned short port) { - int r = 0; - int skt = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (skt==-1) { - E("socket() failed"); - return -1; - } - do { - struct sockaddr_in si = {0}; - si.sin_family = AF_INET; - si.sin_addr.s_addr = inet_addr("0.0.0.0"); - si.sin_port = htons(port); - r = bind(skt, (struct sockaddr*)&si, sizeof(si)); - if (r) { - E("bind(%u) failed", port); - break; - } - r = listen(skt, 100); - if (r) { - E("listen() failed"); - break; - } - memset(&si, 0, sizeof(si)); - socklen_t len = sizeof(si); - r = getsockname(skt, (struct sockaddr *)&si, &len); - if (r) { - E("getsockname() failed"); - } - A(len==sizeof(si), "internal logic error"); - D("listenning at: %d", ntohs(si.sin_port)); - return skt; - } while (0); - close(skt); - return -1; -} - -static void listen_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { - A(ev->events & EPOLLIN, "internal logic error"); - struct sockaddr_in si = {0}; - socklen_t silen = sizeof(si); - int skt = accept(client->skt, (struct sockaddr*)&si, &silen); - A(skt!=-1, "internal logic error"); - fde_t *server = (fde_t*)calloc(1, sizeof(*server)); - if (!server) { - close(skt); - return; - } - server->skt = skt; - server->on_event = null_event; - struct epoll_event ee = {0}; - ee.events = EPOLLIN | EPOLLERR | EPOLLHUP | EPOLLRDHUP; - ee.data.ptr = server; - A(0==epoll_ctl(ep->ep, EPOLL_CTL_ADD, skt, &ee), ""); -} - -static void null_event(ep_t *ep, struct epoll_event *ev, fde_t *client) { - if (ev->events & EPOLLIN) { - char buf[8192]; - int n = recv(client->skt, buf, sizeof(buf), 0); - A(n>=0 && n<=sizeof(buf), "internal logic error:[%d]", n); - A(n==fwrite(buf, 1, n, stdout), "internal logic error"); - } - if (ev->events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP)) { - A(0==pthread_mutex_lock(&ep->lock), ""); - A(0==epoll_ctl(ep->ep, EPOLL_CTL_DEL, client->skt, NULL), ""); - A(0==pthread_mutex_unlock(&ep->lock), ""); - close(client->skt); - client->skt = -1; - client->on_event = NULL; - free(client); - } -} - diff --git a/examples/c/makefile b/examples/c/makefile index c85eb4adc515e5fb4e875b8e8e955222bc09190e..244d13fad74d7b9a124102b47d7c7c642a640438 100644 --- a/examples/c/makefile +++ b/examples/c/makefile @@ -7,22 +7,21 @@ LFLAGS = '-Wl,-rpath,/usr/local/taos/driver/' -ltaos -lpthread -lm -lrt CFLAGS = -O3 -g -Wall -Wno-deprecated -fPIC -Wno-unused-result -Wconversion \ -Wno-char-subscripts -D_REENTRANT -Wno-format -D_REENTRANT -DLINUX \ -Wno-unused-function -D_M_X64 -I/usr/local/taos/include -std=gnu99 \ - -I../../../deps/cJson/inc + -I/usr/local/include/cjson all: $(TARGET) exe: gcc $(CFLAGS) ./asyncdemo.c -o $(ROOT)asyncdemo $(LFLAGS) gcc $(CFLAGS) ./demo.c -o $(ROOT)demo $(LFLAGS) gcc $(CFLAGS) ./prepare.c -o $(ROOT)prepare $(LFLAGS) - gcc $(CFLAGS) ./stream.c -o $(ROOT)stream $(LFLAGS) - gcc $(CFLAGS) ./subscribe.c -o $(ROOT)subscribe $(LFLAGS) - gcc $(CFLAGS) ./apitest.c -o $(ROOT)apitest $(LFLAGS) + gcc $(CFLAGS) ./stream_demo.c -o $(ROOT)stream_demo $(LFLAGS) + gcc $(CFLAGS) ./tmq.c -o $(ROOT)tmq $(LFLAGS) + gcc $(CFLAGS) ./schemaless.c -o $(ROOT)schemaless $(LFLAGS) clean: rm $(ROOT)asyncdemo rm $(ROOT)demo rm $(ROOT)prepare - rm $(ROOT)batchprepare - rm $(ROOT)stream - rm $(ROOT)subscribe - rm $(ROOT)apitest + rm $(ROOT)stream_demo + rm $(ROOT)tmq + rm $(ROOT)schemaless diff --git a/examples/c/prepare.c b/examples/c/prepare.c index f0341cfe127d4eadae29ae40a43c35c35ec2f492..4f05b6fb4cbc648bfde10b04be7388b14b37297e 100644 --- a/examples/c/prepare.c +++ b/examples/c/prepare.c @@ -4,7 +4,7 @@ #include #include #include -#include "../../../include/client/taos.h" +#include "taos.h" void taosMsleep(int mseconds); @@ -70,70 +70,89 @@ int main(int argc, char *argv[]) char blob[80]; } v = {0}; + int32_t boolLen = sizeof(int8_t); + int32_t sintLen = sizeof(int16_t); + int32_t intLen = sizeof(int32_t); + int32_t bintLen = sizeof(int64_t); + int32_t floatLen = sizeof(float); + int32_t doubleLen = sizeof(double); + int32_t binLen = sizeof(v.bin); + int32_t ncharLen = 30; + stmt = taos_stmt_init(taos); - TAOS_BIND params[10]; + TAOS_MULTI_BIND params[10]; params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; params[0].buffer_length = sizeof(v.ts); params[0].buffer = &v.ts; - params[0].length = ¶ms[0].buffer_length; + params[0].length = &bintLen; params[0].is_null = NULL; + params[0].num = 1; params[1].buffer_type = TSDB_DATA_TYPE_BOOL; params[1].buffer_length = sizeof(v.b); params[1].buffer = &v.b; - params[1].length = ¶ms[1].buffer_length; + params[1].length = &boolLen; params[1].is_null = NULL; + params[1].num = 1; params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; params[2].buffer_length = sizeof(v.v1); params[2].buffer = &v.v1; - params[2].length = ¶ms[2].buffer_length; + params[2].length = &boolLen; params[2].is_null = NULL; + params[2].num = 1; params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; params[3].buffer_length = sizeof(v.v2); params[3].buffer = &v.v2; - params[3].length = ¶ms[3].buffer_length; + params[3].length = &sintLen; params[3].is_null = NULL; + params[3].num = 1; params[4].buffer_type = TSDB_DATA_TYPE_INT; params[4].buffer_length = sizeof(v.v4); params[4].buffer = &v.v4; - params[4].length = ¶ms[4].buffer_length; + params[4].length = &intLen; params[4].is_null = NULL; + params[4].num = 1; params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; params[5].buffer_length = sizeof(v.v8); params[5].buffer = &v.v8; - params[5].length = ¶ms[5].buffer_length; + params[5].length = &bintLen; params[5].is_null = NULL; + params[5].num = 1; params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; params[6].buffer_length = sizeof(v.f4); params[6].buffer = &v.f4; - params[6].length = ¶ms[6].buffer_length; + params[6].length = &floatLen; params[6].is_null = NULL; + params[6].num = 1; params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; params[7].buffer_length = sizeof(v.f8); params[7].buffer = &v.f8; - params[7].length = ¶ms[7].buffer_length; + params[7].length = &doubleLen; params[7].is_null = NULL; + params[7].num = 1; params[8].buffer_type = TSDB_DATA_TYPE_BINARY; params[8].buffer_length = sizeof(v.bin); params[8].buffer = v.bin; - params[8].length = ¶ms[8].buffer_length; + params[8].length = &binLen; params[8].is_null = NULL; + params[8].num = 1; strcpy(v.blob, "一二三四五六七八九十"); params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; - params[9].buffer_length = strlen(v.blob); + params[9].buffer_length = sizeof(v.blob); params[9].buffer = v.blob; - params[9].length = ¶ms[9].buffer_length; + params[9].length = &ncharLen; params[9].is_null = NULL; + params[9].num = 1; - int is_null = 1; + char is_null = 1; sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt_prepare(stmt, sql, 0); @@ -153,7 +172,7 @@ int main(int argc, char *argv[]) v.v8 = (int64_t)(i * 8); v.f4 = (float)(i * 40); v.f8 = (double)(i * 80); - for (int j = 0; j < sizeof(v.bin) - 1; ++j) { + for (int j = 0; j < sizeof(v.bin); ++j) { v.bin[j] = (char)(i + '0'); } diff --git a/examples/c/schemaless.c b/examples/c/schemaless.c index 99aa361b0a8801ff72467ed6f644262998f3c5b4..328a663ae79903e742a0c9650e8e947d6beb43d3 100644 --- a/examples/c/schemaless.c +++ b/examples/c/schemaless.c @@ -1,32 +1,17 @@ -#include "../../../include/client/taos.h" -#include "os.h" -#include "taoserror.h" - +#include "taos.h" #include #include #include #include #include +#include +#include + int numSuperTables = 8; int numChildTables = 4; int numRowsPerChildTable = 2048; -void shuffle(char**lines, size_t n) -{ - if (n > 1) - { - size_t i; - for (i = 0; i < n - 1; i++) - { - size_t j = i + taosRand() / (RAND_MAX / (n - i) + 1); - char* t = lines[j]; - lines[j] = lines[i]; - lines[i] = t; - } - } -} - static int64_t getTimeInUs() { struct timeval systemTime; gettimeofday(&systemTime, NULL); @@ -46,7 +31,7 @@ int main(int argc, char* argv[]) { exit(1); } - char* info = taos_get_server_info(taos); + const char* info = taos_get_server_info(taos); printf("server info: %s\n", info); info = taos_get_client_info(taos); printf("client info: %s\n", info); @@ -61,9 +46,10 @@ int main(int argc, char* argv[]) { time_t ct = time(0); int64_t ts = ct * 1000; - char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms"; + char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %" PRId64; - char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*)); + int lineNum = numSuperTables * numChildTables * numRowsPerChildTable; + char** lines = calloc((size_t)lineNum, sizeof(char*)); int l = 0; for (int i = 0; i < numSuperTables; ++i) { for (int j = 0; j < numChildTables; ++j) { @@ -75,13 +61,13 @@ int main(int argc, char* argv[]) { } } } - //shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable); printf("%s\n", "begin taos_insert_lines"); int64_t begin = getTimeInUs(); - int32_t code = taos_insert_lines(taos, lines, numSuperTables * numChildTables * numRowsPerChildTable); + TAOS_RES *res = taos_schemaless_insert(taos, lines, lineNum, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); int64_t end = getTimeInUs(); - printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin); + printf("code: %s. time used: %" PRId64 "\n", taos_errstr(res), end-begin); + taos_free_result(res); return 0; } diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index 46c8297fbac8af6ec7fc3558d690313231569711..46107c7e130bccfc314d816c1ee4b1071803d9da 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -108,10 +108,13 @@ int32_t create_stream() { } int main(int argc, char* argv[]) { - int code; if (argc > 1) { printf("env init\n"); - code = init_env(); + int code = init_env(); + if (code) { + return code; + } + } create_stream(); } diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 1147671ea18484468b22f0bcb30211909aea06b7..71e571d4dd06cb6bd1fc51eb6afba958dfc52bae 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -21,9 +21,6 @@ #include "taos.h" static int running = 1; -static char dbName[64] = "tmqdb"; -static char stbName[64] = "stb"; -static char topicName[64] = "topicname"; static int32_t msg_process(TAOS_RES* msg) { char buf[1024]; @@ -43,7 +40,7 @@ static int32_t msg_process(TAOS_RES* msg) { TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); - int32_t* length = taos_fetch_lengths(msg); + //int32_t* length = taos_fetch_lengths(msg); int32_t precision = taos_result_precision(msg); rows++; taos_print_row(buf, row, fields, numOfFields); @@ -62,6 +59,13 @@ static int32_t init_env() { TAOS_RES* pRes; // drop database if exists printf("create database\n"); + pRes = taos_query(pConn, "drop topic topicname"); + if (taos_errno(pRes) != 0) { + printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "drop database if exists tmqdb"); if (taos_errno(pRes) != 0) { printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); @@ -249,7 +253,7 @@ int main(int argc, char* argv[]) { tmq_t* tmq = build_consumer(); if (NULL == tmq) { - fprintf(stderr, "%% build_consumer() fail!\n"); + fprintf(stderr, "build_consumer() fail!\n"); return -1; } @@ -259,7 +263,7 @@ int main(int argc, char* argv[]) { } if ((code = tmq_subscribe(tmq, topic_list))) { - fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); + fprintf(stderr, "Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); } tmq_list_destroy(topic_list); @@ -267,9 +271,9 @@ int main(int argc, char* argv[]) { code = tmq_consumer_close(tmq); if (code) { - fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + fprintf(stderr, "Failed to close consumer: %s\n", tmq_err2str(code)); } else { - fprintf(stderr, "%% Consumer closed\n"); + fprintf(stderr, "Consumer closed\n"); } return 0; diff --git a/include/client/taos.h b/include/client/taos.h index 2940f1dfd01864cfa7f2107a6ed421fbb0bc20f8..379363c51de7cdd11d8bd050b515ae21d1fa4858 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -149,7 +149,7 @@ DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +DLL_EXPORT const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid); @@ -185,6 +185,7 @@ DLL_EXPORT void taos_kill_query(TAOS *taos); DLL_EXPORT int taos_field_count(TAOS_RES *res); DLL_EXPORT int taos_num_fields(TAOS_RES *res); DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int64_t taos_affected_rows64(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 77f1879b81378aa0bc1bb3dfb0cafd85b792dc5e..19bd511c1faeeb9f2f9cddb5fcd831c83f522476 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -195,6 +195,7 @@ typedef struct SDataBlockInfo { uint32_t capacity; SBlockID id; int16_t hasVarCol; + int16_t dataLoad; // denote if the data is loaded or not // TODO: optimize and remove following int64_t version; // used for stream, and need serialization @@ -249,10 +250,11 @@ typedef struct SColumnInfoData { typedef struct SQueryTableDataCond { uint64_t suid; - int32_t order; // desc|asc order to iterate the data block + int32_t order; // desc|asc order to iterate the data block int32_t numOfCols; SColumnInfo* colList; - int32_t type; // data block load type: + int32_t* pSlotList; // the column output destation slot, and it may be null + int32_t type; // data block load type: STimeWindow twindows; int64_t startVersion; int64_t endVersion; @@ -338,7 +340,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - int32_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -347,7 +349,7 @@ typedef struct { double d; float f; }; - int32_t length; + size_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index db6ca65cf05976a33e74a5d497df6a44aead5bc1..d9b8ae266b358507a8b396180b67c1bb536bdf18 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -41,9 +41,9 @@ typedef struct SBlockOrderInfo { BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -#define colDataSetNotNull_f(bm_, r_) \ - do { \ - BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ +#define colDataClearNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) &= ((char)(~(1u << (7u - BitPos(r_))))); \ } while (0) #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) @@ -151,9 +151,6 @@ static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, ui for (int32_t i = start; i < start + nRows; ++i) { colDataSetNull_f(pColumnInfoData->nullbitmap, i); } - - int32_t bytes = pColumnInfoData->info.bytes; - memset(pColumnInfoData->pData + start * bytes, 0, nRows * bytes); } pColumnInfoData->hasNull = true; @@ -234,9 +231,11 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void blockDataCleanup(SSDataBlock* pDataBlock); +void blockDataEmpty(SSDataBlock* pDataBlock); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); @@ -266,7 +265,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 21271048825e30302c0b11d1414b47b739597f89..e0aacbfec9db8804b4003417689f404b78549afb 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -44,18 +44,38 @@ typedef struct SColData SColData; #define HAS_VALUE ((uint8_t)0x4) // bitmap ================================ -const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000000, 0b00000100, 0b00001000, 2}, - {0b00000000, 0b00010000, 0b00100000, 4}, - {0b00000000, 0b01000000, 0b10000000, 6}}; - -#define N1(n) ((((uint8_t)1) << (n)) - 1) -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) -#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) -#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, + 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + +const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; + +#define ONE ((uint8_t)1) +#define THREE ((uint8_t)3) +#define DIV_8(i) ((i) >> 3) +#define MOD_8(i) ((i)&7) +#define DIV_4(i) ((i) >> 2) +#define MOD_4(i) ((i)&3) +#define MOD_4_TIME_2(i) (MOD_4(i) << 1) +#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) +#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) +#define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) +#define SET_BIT1_EX(p, i, v) \ + do { \ + if (MOD_8(i) == 0) { \ + (p)[DIV_8(i)] = 0; \ + } \ + SET_BIT1(p, i, v); \ + } while (0) +#define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) +#define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) +#define SET_BIT2_EX(p, i, v) \ + do { \ + if (MOD_4(i) == 0) { \ + (p)[DIV_4(i)] = 0; \ + } \ + SET_BIT2(p, i, v); \ + } while (0) +#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) // SBuffer ================================ struct SBuffer { @@ -84,7 +104,7 @@ int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow); +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow); void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); void tRowDestroy(SRow *pRow); void tRowSort(SArray *aRowP); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 005cf36d5ea11e8adaeb61a95a9536c1af746982..92672311d0661af86fbf2026d23ac1cb8f5a345c 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,6 +64,11 @@ extern int32_t tsNumOfSnodeStreamThreads; extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsRpcQueueMemoryAllowed; +// sync raft +extern int32_t tsElectInterval; +extern int32_t tsHeartbeatInterval; +extern int32_t tsHeartbeatTimeout; + // monitor extern bool tsEnableMonitor; extern int32_t tsMonitorInterval; @@ -126,9 +131,9 @@ extern char tsUdfdResFuncs[]; extern char tsUdfdLdLibPath[]; // schemaless -extern char tsSmlChildTableName[]; -extern char tsSmlTagName[]; -extern bool tsSmlDataFormat; +extern char tsSmlChildTableName[]; +extern char tsSmlTagName[]; +extern bool tsSmlDataFormat; extern int32_t tsSmlBatchSize; // wal @@ -146,7 +151,7 @@ extern int32_t tsUptimeInterval; extern int32_t tsRpcRetryLimit; extern int32_t tsRpcRetryInterval; -//#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) +// #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ad992fd9dbf50f9c789c636238b3d005de788fa8..603d6cfd67a13cb38d88ddcb148897196907f9f1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -66,6 +66,15 @@ extern int32_t tMsgDict[]; typedef uint16_t tmsg_t; +static inline bool vnodeIsMsgBlock(tmsg_t type) { + return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); +} + +static inline bool syncUtilUserCommit(tmsg_t msgType) { + return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; +} + /* ------------------------ OTHER DEFINITIONS ------------------------ */ // IE type #define TSDB_IE_TYPE_SEC 1 @@ -498,6 +507,8 @@ typedef struct { char* pComment; char* pAst1; char* pAst2; + int64_t deleteMark1; + int64_t deleteMark2; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -1401,8 +1412,8 @@ typedef struct { int8_t streamBlockType; int32_t compLen; int32_t numOfBlocks; - int32_t numOfRows; - int32_t numOfCols; + int64_t numOfRows; // from int32_t change to int64_t + int64_t numOfCols; int64_t skey; int64_t ekey; int64_t version; // for stream @@ -2013,6 +2024,7 @@ typedef struct { typedef struct { int64_t maxdelay[2]; int64_t watermark[2]; + int64_t deleteMark[2]; int32_t qmsgLen[2]; char* qmsg[2]; // pAst:qmsg:SRetention => trigger aggr task1/2 } SRSmaParam; @@ -2069,8 +2081,9 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); -int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +void tDestroySVCreateTbReq(SVCreateTbReq* pReq, int32_t flags); static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { if (NULL == req) { @@ -2738,6 +2751,7 @@ typedef struct { char* tagsFilter; char* sql; char* ast; + int64_t deleteMark; } SMCreateSmaReq; int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); @@ -3264,6 +3278,17 @@ void tDestroySSubmitRsp2(SSubmitRsp2* pRsp, int32_t flag); #define TSDB_MSG_FLG_ENCODE 0x1 #define TSDB_MSG_FLG_DECODE 0x2 +typedef struct { + union { + struct { + void* msgStr; + int32_t msgLen; + int64_t ver; + }; + void* pDataBlock; + }; +} SPackedData; + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index b5b997dac0a18c6ca83e07d5f5c597004df8a913..32d00bb4227c8b153e4112851ba6004b5ab7c88d 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -43,7 +43,6 @@ typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); typedef void (*SendRspFp)(SRpcMsg* pMsg); -typedef void (*SendRedirectRspFp)(SRpcMsg* pMsg, const SEpSet* pNewEpSet); typedef void (*RegisterBrokenLinkArgFp)(SRpcMsg* pMsg); typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type); typedef void (*ReportStartup)(const char* name, const char* desc); @@ -55,7 +54,6 @@ typedef struct { GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; SendRspFp sendRspFp; - SendRedirectRspFp sendRedirectRspFp; RegisterBrokenLinkArgFp registerBrokenLinkArgFp; ReleaseHandleFp releaseHandleFp; ReportStartup reportStartupFp; @@ -66,7 +64,6 @@ int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg); int32_t tmsgGetQueueSize(const SMsgCb* msgcb, int32_t vgId, EQueueType qtype); int32_t tmsgSendReq(const SEpSet* epSet, SRpcMsg* pMsg); void tmsgSendRsp(SRpcMsg* pMsg); -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index ccf91600dbabb7fdf84f74b367748fd037882896..7533cfbb023dd33d47cf7020c01c547e10d5df72 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -94,59 +94,59 @@ #define TK_TSDB_PAGESIZE 76 #define TK_PRECISION 77 #define TK_REPLICA 78 -#define TK_STRICT 79 -#define TK_VGROUPS 80 -#define TK_SINGLE_STABLE 81 -#define TK_RETENTIONS 82 -#define TK_SCHEMALESS 83 -#define TK_WAL_LEVEL 84 -#define TK_WAL_FSYNC_PERIOD 85 -#define TK_WAL_RETENTION_PERIOD 86 -#define TK_WAL_RETENTION_SIZE 87 -#define TK_WAL_ROLL_PERIOD 88 -#define TK_WAL_SEGMENT_SIZE 89 -#define TK_STT_TRIGGER 90 -#define TK_TABLE_PREFIX 91 -#define TK_TABLE_SUFFIX 92 -#define TK_NK_COLON 93 -#define TK_MAX_SPEED 94 -#define TK_TABLE 95 -#define TK_NK_LP 96 -#define TK_NK_RP 97 -#define TK_STABLE 98 -#define TK_ADD 99 -#define TK_COLUMN 100 -#define TK_MODIFY 101 -#define TK_RENAME 102 -#define TK_TAG 103 -#define TK_SET 104 -#define TK_NK_EQ 105 -#define TK_USING 106 -#define TK_TAGS 107 -#define TK_COMMENT 108 -#define TK_BOOL 109 -#define TK_TINYINT 110 -#define TK_SMALLINT 111 -#define TK_INT 112 -#define TK_INTEGER 113 -#define TK_BIGINT 114 -#define TK_FLOAT 115 -#define TK_DOUBLE 116 -#define TK_BINARY 117 -#define TK_TIMESTAMP 118 -#define TK_NCHAR 119 -#define TK_UNSIGNED 120 -#define TK_JSON 121 -#define TK_VARCHAR 122 -#define TK_MEDIUMBLOB 123 -#define TK_BLOB 124 -#define TK_VARBINARY 125 -#define TK_DECIMAL 126 -#define TK_MAX_DELAY 127 -#define TK_WATERMARK 128 -#define TK_ROLLUP 129 -#define TK_TTL 130 -#define TK_SMA 131 +#define TK_VGROUPS 79 +#define TK_SINGLE_STABLE 80 +#define TK_RETENTIONS 81 +#define TK_SCHEMALESS 82 +#define TK_WAL_LEVEL 83 +#define TK_WAL_FSYNC_PERIOD 84 +#define TK_WAL_RETENTION_PERIOD 85 +#define TK_WAL_RETENTION_SIZE 86 +#define TK_WAL_ROLL_PERIOD 87 +#define TK_WAL_SEGMENT_SIZE 88 +#define TK_STT_TRIGGER 89 +#define TK_TABLE_PREFIX 90 +#define TK_TABLE_SUFFIX 91 +#define TK_NK_COLON 92 +#define TK_MAX_SPEED 93 +#define TK_TABLE 94 +#define TK_NK_LP 95 +#define TK_NK_RP 96 +#define TK_STABLE 97 +#define TK_ADD 98 +#define TK_COLUMN 99 +#define TK_MODIFY 100 +#define TK_RENAME 101 +#define TK_TAG 102 +#define TK_SET 103 +#define TK_NK_EQ 104 +#define TK_USING 105 +#define TK_TAGS 106 +#define TK_COMMENT 107 +#define TK_BOOL 108 +#define TK_TINYINT 109 +#define TK_SMALLINT 110 +#define TK_INT 111 +#define TK_INTEGER 112 +#define TK_BIGINT 113 +#define TK_FLOAT 114 +#define TK_DOUBLE 115 +#define TK_BINARY 116 +#define TK_TIMESTAMP 117 +#define TK_NCHAR 118 +#define TK_UNSIGNED 119 +#define TK_JSON 120 +#define TK_VARCHAR 121 +#define TK_MEDIUMBLOB 122 +#define TK_BLOB 123 +#define TK_VARBINARY 124 +#define TK_DECIMAL 125 +#define TK_MAX_DELAY 126 +#define TK_WATERMARK 127 +#define TK_ROLLUP 128 +#define TK_TTL 129 +#define TK_SMA 130 +#define TK_DELETE_MARK 131 #define TK_FIRST 132 #define TK_LAST 133 #define TK_SHOW 134 @@ -266,76 +266,79 @@ #define TK_BY 248 #define TK_SESSION 249 #define TK_STATE_WINDOW 250 -#define TK_SLIDING 251 -#define TK_FILL 252 -#define TK_VALUE 253 -#define TK_NONE 254 -#define TK_PREV 255 -#define TK_LINEAR 256 -#define TK_NEXT 257 -#define TK_HAVING 258 -#define TK_RANGE 259 -#define TK_EVERY 260 -#define TK_ORDER 261 -#define TK_SLIMIT 262 -#define TK_SOFFSET 263 -#define TK_LIMIT 264 -#define TK_OFFSET 265 -#define TK_ASC 266 -#define TK_NULLS 267 -#define TK_ABORT 268 -#define TK_AFTER 269 -#define TK_ATTACH 270 -#define TK_BEFORE 271 -#define TK_BEGIN 272 -#define TK_BITAND 273 -#define TK_BITNOT 274 -#define TK_BITOR 275 -#define TK_BLOCKS 276 -#define TK_CHANGE 277 -#define TK_COMMA 278 -#define TK_COMPACT 279 -#define TK_CONCAT 280 -#define TK_CONFLICT 281 -#define TK_COPY 282 -#define TK_DEFERRED 283 -#define TK_DELIMITERS 284 -#define TK_DETACH 285 -#define TK_DIVIDE 286 -#define TK_DOT 287 -#define TK_EACH 288 -#define TK_FAIL 289 -#define TK_FILE 290 -#define TK_FOR 291 -#define TK_GLOB 292 -#define TK_ID 293 -#define TK_IMMEDIATE 294 -#define TK_IMPORT 295 -#define TK_INITIALLY 296 -#define TK_INSTEAD 297 -#define TK_ISNULL 298 -#define TK_KEY 299 -#define TK_MODULES 300 -#define TK_NK_BITNOT 301 -#define TK_NK_SEMI 302 -#define TK_NOTNULL 303 -#define TK_OF 304 -#define TK_PLUS 305 -#define TK_PRIVILEGE 306 -#define TK_RAISE 307 -#define TK_REPLACE 308 -#define TK_RESTRICT 309 -#define TK_ROW 310 -#define TK_SEMI 311 -#define TK_STAR 312 -#define TK_STATEMENT 313 -#define TK_STRING 314 -#define TK_TIMES 315 -#define TK_UPDATE 316 -#define TK_VALUES 317 -#define TK_VARIABLE 318 -#define TK_VIEW 319 -#define TK_WAL 320 +#define TK_EVENT_WINDOW 251 +#define TK_START 252 +#define TK_SLIDING 253 +#define TK_FILL 254 +#define TK_VALUE 255 +#define TK_NONE 256 +#define TK_PREV 257 +#define TK_LINEAR 258 +#define TK_NEXT 259 +#define TK_HAVING 260 +#define TK_RANGE 261 +#define TK_EVERY 262 +#define TK_ORDER 263 +#define TK_SLIMIT 264 +#define TK_SOFFSET 265 +#define TK_LIMIT 266 +#define TK_OFFSET 267 +#define TK_ASC 268 +#define TK_NULLS 269 +#define TK_ABORT 270 +#define TK_AFTER 271 +#define TK_ATTACH 272 +#define TK_BEFORE 273 +#define TK_BEGIN 274 +#define TK_BITAND 275 +#define TK_BITNOT 276 +#define TK_BITOR 277 +#define TK_BLOCKS 278 +#define TK_CHANGE 279 +#define TK_COMMA 280 +#define TK_COMPACT 281 +#define TK_CONCAT 282 +#define TK_CONFLICT 283 +#define TK_COPY 284 +#define TK_DEFERRED 285 +#define TK_DELIMITERS 286 +#define TK_DETACH 287 +#define TK_DIVIDE 288 +#define TK_DOT 289 +#define TK_EACH 290 +#define TK_FAIL 291 +#define TK_FILE 292 +#define TK_FOR 293 +#define TK_GLOB 294 +#define TK_ID 295 +#define TK_IMMEDIATE 296 +#define TK_IMPORT 297 +#define TK_INITIALLY 298 +#define TK_INSTEAD 299 +#define TK_ISNULL 300 +#define TK_KEY 301 +#define TK_MODULES 302 +#define TK_NK_BITNOT 303 +#define TK_NK_SEMI 304 +#define TK_NOTNULL 305 +#define TK_OF 306 +#define TK_PLUS 307 +#define TK_PRIVILEGE 308 +#define TK_RAISE 309 +#define TK_REPLACE 310 +#define TK_RESTRICT 311 +#define TK_ROW 312 +#define TK_SEMI 313 +#define TK_STAR 314 +#define TK_STATEMENT 315 +#define TK_STRICT 316 +#define TK_STRING 317 +#define TK_TIMES 318 +#define TK_UPDATE 319 +#define TK_VALUES 320 +#define TK_VARIABLE 321 +#define TK_VIEW 322 +#define TK_WAL 323 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 761ffd0f1c1c9d480c312382897a5b30b52b7335..6350057c1f63b1bfb49694ca635e5e064c11d406 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -278,11 +278,9 @@ typedef struct { #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_INT(_t) ((_t) >= INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) >= INT64_MIN && (_t) <= INT64_MAX) #define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) <= UINT8_MAX) #define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) <= UINT16_MAX) #define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) <= UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) <= UINT64_MAX) #define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) #define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6154882756e0cb738f94fdf566cd9883da766c32..a95db86d3fc2de33b290f9211ce24be6c2549972 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -211,6 +211,8 @@ int32_t catalogGetCachedSTableMeta(SCatalog* pCtg, const SName* pTableName, STab int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); +int32_t catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta); + /** * Force refresh DB's local cached vgroup info. * @param pCtg (input, got with catalogGetHandle) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 8a02f372d1d605b359482b5e27810e6f95488433..ed7cbc812504ae09099abd79020c6b9d761d6674 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -68,7 +68,7 @@ typedef struct SInputData { typedef struct SOutputData { int32_t numOfBlocks; - int32_t numOfRows; + int64_t numOfRows; // int32_t changed to int64_t int32_t numOfCols; int8_t compressed; char* pData; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 412b4b4cf6c950113a2f7e6d319692695cde0c07..4d9c0866c31c7ee5c11c58754c399d273796e0af 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -190,7 +190,9 @@ int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq); +// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); +// +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 13a8370dfd08e1931aec9859dd078d82a670a6eb..32b8cc7389b96e7479697b2c864e98543d92e9a1 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -137,22 +137,22 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; - SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - int32_t offset; - struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaries; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; - struct SExprInfo *pExpr; - struct SSDataBlock *pSrcBlock; - struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity - SSerializeDataHandle saveHandle; - bool isStream; - - char udfName[TSDB_FUNC_NAME_LEN]; + // input parameter, e.g., top(k, 20), the number of results of top query is kept in param + SFunctParam *param; + // corresponding output buffer for timestamp of each result, e.g., diff/csum + SColumnInfoData *pTsOutput; + int32_t offset; + SResultRowEntryInfo *resultInfo; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + struct SExprInfo *pExpr; + struct SSDataBlock *pSrcBlock; + struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity + SSerializeDataHandle saveHandle; + char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; typedef struct tExprNode { @@ -163,6 +163,7 @@ typedef struct tExprNode { int32_t functionId; int32_t num; struct SFunctionNode *pFunctNode; + int32_t functionType; } _function; struct { @@ -182,7 +183,6 @@ struct SScalarParam { }; void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); -//int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock); bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); @@ -194,32 +194,6 @@ typedef struct SPoint { int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, int32_t inputType); -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// udf api -/** - * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf - * @return error code - */ -int32_t udfcOpen(); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(); - -/** - * start udfd that serves udf function invocation under dnode startDnodeId - * @param startDnodeId - * @return - */ -int32_t udfStartUdfd(int32_t startDnodeId); -/** - * stop udfd - * @return - */ -int32_t udfStopUdfd(); - #ifdef __cplusplus } #endif diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 81f63537e59ee7f51e01a0bc5c9527601be83300..9ca6a7a9fafe464f1f82682b9e3599d15b6dff79 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -130,6 +130,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_GROUP_KEY, FUNCTION_TYPE_CACHE_LAST_ROW, FUNCTION_TYPE_CACHE_LAST, + FUNCTION_TYPE_TABLE_COUNT, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 5f57e203b9c04de0efa34ede4eaeb57503e18db3..1b1339340b1dc13d85740e953400ac6c0ee85f40 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -228,7 +228,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR newSize = 8; } - while (newSize < data->varLenCol.payloadLen + dataLen) { + while (newSize < (uint32_t)(data->varLenCol.payloadLen + dataLen)) { newSize = newSize * UDF_MEMORY_EXP_GROWTH; } @@ -248,7 +248,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR data->varLenCol.payloadLen += dataLen; } } - data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow + 1) : data->numOfRows; + data->numOfRows = ((int32_t)(currentRow + 1) > data->numOfRows) ? (int32_t)(currentRow + 1) : data->numOfRows; return 0; } diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index 31cc53bb9f01a062db95fcdcc8b7704c23e12660..b71d50d43cc59988407576c1c1e0b9c2bce8fa3b 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -85,6 +85,32 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, int32_t cleanUpUdfs(); +/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// udf api +/** + * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +/** + * start udfd that serves udf function invocation under dnode startDnodeId + * @param startDnodeId + * @return + */ +int32_t udfStartUdfd(int32_t startDnodeId); +/** + * stop udfd + * @return + */ +int32_t udfStopUdfd(); + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index e69c6e747dccaf5eddaf979bdea037ee1adedbfc..b1054e7b859c8ba9231ee1c942f12d7c1ec6b97c 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -133,6 +133,9 @@ typedef struct STableOptions { SNodeList* pWatermark; int64_t watermark1; int64_t watermark2; + SNodeList* pDeleteMark; + int64_t deleteMark1; + int64_t deleteMark2; SNodeList* pRollupFuncs; int32_t ttl; SNodeList* pSma; @@ -383,6 +386,7 @@ typedef struct SStreamOptions { int8_t triggerType; SNode* pDelay; SNode* pWatermark; + SNode* pDeleteMark; int8_t fillHistory; int8_t ignoreExpired; } SStreamOptions; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index cd7cedb6c1901eb8c25c8a610f0f679fa64c2285..1f266cd0ef612c2c7142b62e3719578e5832e9c4 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -60,6 +60,12 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) +#define NODES_DESTORY_NODE(node) \ + do { \ + nodesDestroyNode((node)); \ + (node) = NULL; \ + } while (0) + #define NODES_DESTORY_LIST(list) \ do { \ nodesDestroyList((list)); \ @@ -106,6 +112,7 @@ typedef enum ENodeType { QUERY_NODE_COLUMN_REF, QUERY_NODE_WHEN_THEN, QUERY_NODE_CASE_WHEN, + QUERY_NODE_EVENT_WINDOW, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR = 100, @@ -258,7 +265,10 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, QUERY_NODE_PHYSICAL_PLAN_DELETE, QUERY_NODE_PHYSICAL_SUBPLAN, - QUERY_NODE_PHYSICAL_PLAN + QUERY_NODE_PHYSICAL_PLAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, + QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 6a6fd50f2a47656856e5fd5936e3eed01481503f..6f700c75edf82a9eddb23ef82f0dade1c89680b2 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -62,7 +62,8 @@ typedef enum EScanType { SCAN_TYPE_STREAM, SCAN_TYPE_TABLE_MERGE, SCAN_TYPE_BLOCK_INFO, - SCAN_TYPE_LAST_ROW + SCAN_TYPE_LAST_ROW, + SCAN_TYPE_TABLE_COUNT } EScanType; typedef struct SScanLogicNode { @@ -90,6 +91,7 @@ typedef struct SScanLogicNode { SNode* pTagIndexCond; int8_t triggerType; int64_t watermark; + int64_t deleteMark; int8_t igExpired; SArray* pSmaIndexes; SNodeList* pGroupTags; @@ -183,7 +185,12 @@ typedef struct SMergeLogicNode { bool groupSort; } SMergeLogicNode; -typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; +typedef enum EWindowType { + WINDOW_TYPE_INTERVAL = 1, + WINDOW_TYPE_SESSION, + WINDOW_TYPE_STATE, + WINDOW_TYPE_EVENT +} EWindowType; typedef enum EWindowAlgorithm { INTERVAL_ALGO_HASH = 1, @@ -210,8 +217,11 @@ typedef struct SWindowLogicNode { SNode* pTspk; SNode* pTsEnd; SNode* pStateExpr; + SNode* pStartCond; + SNode* pEndCond; int8_t triggerType; int64_t watermark; + int64_t deleteMark; int8_t igExpired; EWindowAlgorithm windowAlgo; EOrder inputTsOrder; @@ -323,6 +333,8 @@ typedef struct SLastRowScanPhysiNode { bool ignoreNull; } SLastRowScanPhysiNode; +typedef SLastRowScanPhysiNode STableCountScanPhysiNode; + typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; SEpSet mgmtEpSet; @@ -437,6 +449,7 @@ typedef struct SWinodwPhysiNode { SNode* pTsEnd; // window end timestamp int8_t triggerType; int64_t watermark; + int64_t deleteMark; int8_t igExpired; EOrder inputTsOrder; EOrder outputTsOrder; @@ -492,6 +505,14 @@ typedef struct SStateWinodwPhysiNode { typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; +typedef struct SEventWinodwPhysiNode { + SWinodwPhysiNode window; + SNode* pStartCond; + SNode* pEndCond; +} SEventWinodwPhysiNode; + +typedef SEventWinodwPhysiNode SStreamEventWinodwPhysiNode; + typedef struct SSortPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 6f7fd4db53f79d3bbae88c3181a7a3d07d7c8211..ec13dda5a415aef7c8eba88e36c01499d6e2d8d6 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -223,6 +223,13 @@ typedef struct SIntervalWindowNode { SNode* pFill; } SIntervalWindowNode; +typedef struct SEventWindowNode { + ENodeType type; // QUERY_NODE_EVENT_WINDOW + SNode* pCol; // timestamp primary key + SNode* pStartCond; + SNode* pEndCond; +} SEventWindowNode; + typedef enum EFillMode { FILL_MODE_NONE = 1, FILL_MODE_VALUE, diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 4ff0bf57c4117eac05092e769f5aefc8c387331c..40bbe275eaa2a9fb9d77f5b89fd44e6f0a5eb576 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -85,11 +84,12 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc, bool reset); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb); +void qDestroyStmtDataBlock(STableDataCxt* pBlock); +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); @@ -107,7 +107,11 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* void qDestroyBoundColInfo(void* pInfo); SQuery* smlInitHandle(); -int32_t smlBindData(SQuery* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 82f5c478c1ec7a83728e18997b1b494c1e627057..f7bd68393e67b6c5a92587bc3fad56a20891b4fd 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -34,6 +34,7 @@ typedef struct SPlanContext { bool showRewrite; int8_t triggerType; int64_t watermark; + int64_t deleteMark; int8_t igExpired; char* pMsg; int32_t msgLen; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 63192812122025b10b85c59af343216df139d1c0..4bd23082c46ff4ff8c4a496b72c1641b095d4b15 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -58,8 +58,7 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) -#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) - +#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema @@ -128,7 +127,8 @@ typedef struct SDBVgInfo { int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT int64_t stateTs; - SHashObj* vgHash; // key:vgId, value:SVgroupInfo + SHashObj* vgHash; // key:vgId, value:SVgroupInfo + SArray* vgArray; } SDBVgInfo; typedef struct SUseDbOutput { @@ -163,6 +163,23 @@ typedef struct STargetInfo { int32_t vgId; } STargetInfo; +typedef struct SBoundColInfo { + int16_t* pColIndex; // bound index => schema index + int32_t numOfCols; + int32_t numOfBound; +} SBoundColInfo; + +typedef struct STableDataCxt { + STableMeta* pMeta; + STSchema* pSchema; + SBoundColInfo boundColsInfo; + SArray* pValues; + SSubmitTbData* pData; + TSKEY lastTs; + bool ordered; + bool duplicateTs; +} STableDataCxt; + typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -238,6 +255,8 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); +void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int64_t)); @@ -260,25 +279,26 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || (_code) == TSDB_CODE_VND_STOPPED) -#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) -#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) (false) // used later +#define SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR || \ + (_code) == TSDB_CODE_VND_STOPPED || (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) +#define SYNC_SELF_LEADER_REDIRECT_ERROR(_code) \ + ((_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_SYN_RESTORING || (_code) == TSDB_CODE_SYN_INTERNAL_ERROR) +#define SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_MNODE_NOT_FOUND) + +#define NO_RET_REDIRECT_ERROR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) -#define NEED_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ - (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || \ - (_code) == TSDB_CODE_APP_NOT_READY || (_code) == TSDB_CODE_RPC_BROKEN_LINK || \ - (_code) == TSDB_CODE_APP_IS_STARTING || (_code) == TSDB_CODE_APP_IS_STOPPING) +#define NEED_REDIRECT_ERROR(_code) \ + (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code)) #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ (_type) == TDMT_MND_DROP_STB) -#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \ - SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ - SYNC_OTHER_LEADER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_APP_NOT_READY) +#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ + (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ + SYNC_OTHER_LEADER_REDIRECT_ERROR(_code)) #define REQUEST_TOTAL_EXEC_TIMES 2 diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 59f030a60c651a118b4f956996eb8e60ce42cd87..8fdac0da7fe3758a7ba335ee11c0709f2cf56e83 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,7 +35,7 @@ typedef struct STdbState { TTB* pFillStateDb; // todo refactor TTB* pSessionStateDb; TTB* pParNameDb; - TXN txn; + TXN* txn; } STdbState; // incremental state storage diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0196dcb0a818479b81f14fd1755d37fd11ffee52..a2a512fb0713287edae2c52eb2ff7250e15c6f03 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -103,6 +103,7 @@ typedef struct { int8_t type; } SStreamQueueItem; +#if 0 typedef struct { int8_t type; int64_t ver; @@ -116,6 +117,21 @@ typedef struct { SArray* dataRefs; // SArray SArray* reqs; // SArray } SStreamMergedSubmit; +#endif + +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SPackedData submit; +} SStreamDataSubmit2; + +typedef struct { + int8_t type; + int64_t ver; + SArray* dataRefs; // SArray + SArray* submits; // SArray +} SStreamMergedSubmit2; typedef struct { int8_t type; @@ -219,11 +235,11 @@ static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { } } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; @@ -355,14 +371,15 @@ void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); if (pSubmitClone == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); + qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, + pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE || @@ -620,7 +637,7 @@ typedef struct SStreamMeta { SHashObj* pTasks; SHashObj* pRecoverStatus; void* ahandle; - TXN txn; + TXN* txn; FTaskExpand* expandFunc; int32_t vgId; SRWLatch lock; diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 4741f7f3d8227c28b1979ca6925c52ee02863a4e..a13d203889fbd36c383f622d6e0d6d29d91d39a6 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -25,7 +25,7 @@ extern "C" { #include "tlrucache.h" #include "tmsgcb.h" -#define SYNC_RESP_TTL_MS 10000000 +#define SYNC_RESP_TTL_MS 30000 #define SYNC_SPEED_UP_HB_TIMER 400 #define SYNC_SPEED_UP_AFTER_MS (1000 * 20) #define SYNC_SLOW_DOWN_RANGE 100 @@ -43,10 +43,11 @@ extern "C" { #define SYNC_MAX_RETRY_BACKOFF 5 #define SYNC_LOG_REPL_RETRY_WAIT_MS 100 #define SYNC_APPEND_ENTRIES_TIMEOUT_MS 10000 -#define SYNC_HEART_TIMEOUT_MS 1000 * 8 +#define SYNC_HEART_TIMEOUT_MS 1000 * 15 #define SYNC_HEARTBEAT_SLOW_MS 1500 #define SYNC_HEARTBEAT_REPLY_SLOW_MS 1500 +#define SYNC_SNAP_RESEND_MS 1000 * 60 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 @@ -138,8 +139,8 @@ typedef struct SSnapshotMeta { typedef struct SSyncFSM { void* data; - void (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); - void (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); + int32_t (*FpPreCommitCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRollBackCb)(const struct SSyncFSM* pFsm, SRpcMsg* pMsg, const SFsmCbMeta* pMeta); void (*FpRestoreFinishCb)(const struct SSyncFSM* pFsm); @@ -229,7 +230,7 @@ int64_t syncOpen(SSyncInfo* pSyncInfo); int32_t syncStart(int64_t rid); void syncStop(int64_t rid); void syncPreStop(int64_t rid); -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); @@ -239,6 +240,7 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm); bool syncIsReadyForRead(int64_t rid); bool syncSnapshotSending(int64_t rid); bool syncSnapshotRecving(int64_t rid); +int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq); SSyncState syncGetState(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index d761813db1b5cf10058ada050e2647c184dd5e1c..de3c2a9f52f7f7743ae58f57c0bc53d1cc0d2991 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -72,24 +72,26 @@ typedef struct SRpcMsg { typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcFFfp)(tmsg_t msgType); typedef void (*RpcDfp)(void *ahandle); typedef struct SRpcInit { char localFqdn[TSDB_FQDN_LEN]; - uint16_t localPort; // local port - char *label; // for debug purpose - int32_t numOfThreads; // number of threads to handle connections - int32_t sessions; // number of sessions allowed - int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS - int32_t idleTime; // milliseconds, 0 means idle timer is disabled - int32_t retryLimit; // retry limit - int32_t retryInterval; // retry interval ms + uint16_t localPort; // local port + char *label; // for debug purpose + int32_t numOfThreads; // number of threads to handle connections + int32_t sessions; // number of sessions allowed + int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS + int32_t idleTime; // milliseconds, 0 means idle timer is disabled int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval int64_t retryMaxTimouet; + int32_t failFastThreshold; + int32_t failFastInterval; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size int8_t encryption; // encrypt or not @@ -107,6 +109,8 @@ typedef struct SRpcInit { // destroy client ahandle; RpcDfp dfp; + // fail fast fp + RpcFFfp ffp; void *parent; } SRpcInit; diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 936f11c41ac688a263ab1bd3f379d97ba2f022f1..a1ae1e429dd5dbb18f6521b263576e7096482327 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -33,16 +33,16 @@ extern "C" { #define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", DEBUG_TRACE, wDebugFlag, __VA_ARGS__); }} // clang-format on -#define WAL_PROTO_VER 0 -#define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) -#define WAL_LOG_SUFFIX "log" -#define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL -#define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) +#define WAL_PROTO_VER 0 +#define WAL_NOSUFFIX_LEN 20 +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" +#define WAL_INDEX_SUFFIX "idx" +#define WAL_REFRESH_MS 1000 +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_MAGIC 0xFAFBFCFDF4F3F2F1ULL +#define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3) typedef enum { TAOS_WAL_WRITE = 1, @@ -107,6 +107,8 @@ typedef struct SWal { TdFilePtr pIdxFile; int32_t writeCur; SArray *fileInfoSet; // SArray + // gc + SArray *toDeleteFiles; // SArray // status int64_t totSize; int64_t lastRollSeq; diff --git a/include/os/os.h b/include/os/os.h index 0688eeb9addf920b4a27a7fac0dcab0ec4735144..b27fa84406b843f15ba99d52f7d9fde8580a0899 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -27,6 +27,7 @@ extern "C" { #if !defined(WINDOWS) #include +#include #include #include #include diff --git a/include/os/osDef.h b/include/os/osDef.h index 0bf9c6184eab2d6084ff18acda7367ac54ff8cd4..c18728c9a74276fe743e911b4301c57baa44e150 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -120,12 +120,6 @@ void syslog(int unused, const char *format, ...); #define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) #define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - #ifndef UNUSED #define UNUSED(x) ((void)(x)) #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 40012f8141d9be3ad4b1d669327e019516448df4..73871602c5e7818f0d9b050280d7326ee39678ef 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -62,6 +62,7 @@ int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char *taosDirName(char *dirname); char *taosDirEntryBaseName(char *dirname); +void taosGetCwd(char *buf, int32_t len); TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a3bd209693a1bb3dc4958173d4f4a1744d13a713..533d989ffc40c2aae35adc02bde6080da52ac219 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -36,7 +36,7 @@ extern int64_t tsStreamMax; extern float tsNumOfCores; extern int64_t tsTotalMemoryKB; extern char *tsProcPath; -extern char tsSIMDEnable; +extern char tsSIMDBuiltins; extern char tsSSE42Enable; extern char tsAVXEnable; extern char tsAVX2Enable; diff --git a/include/os/osFile.h b/include/os/osFile.h index f6759d19a74499cc16e08ad84ff543c7a17d7550..ae77e0f27a26adf679f72be92c235e61b6ed0655 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -88,6 +88,7 @@ int32_t taosFsyncFile(TdFilePtr pFile); int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset); void taosFprintfFile(TdFilePtr pFile, const char *format, ...); int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf); diff --git a/include/os/osString.h b/include/os/osString.h index 4c3fbe46c3870d98ae8b22f4dcf9a6edcb9ab4f7..b609c9d3513177c836c79572d2fd019e9a09cc6b 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -62,7 +62,7 @@ typedef int32_t TdUcs4; int32_t taosUcs4len(TdUcs4 *ucs4); int64_t taosStr2int64(const char *str); -void taosConvInit(void); +int32_t taosConvInit(void); void taosConvDestroy(); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); diff --git a/include/os/osSystem.h b/include/os/osSystem.h index eca984c41a3a0502ba726dfbac228dcc82076298..8f1f5c58d5200d22583dcc7cc8c13663ce070941 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -46,6 +46,29 @@ void taosSetTerminalMode(); int32_t taosGetOldTerminalMode(); void taosResetTerminalMode(); +#if !defined(WINDOWS) +#define taosPrintTrace(flags, level, dflag) \ + { \ + void* array[100]; \ + int32_t size = backtrace(array, 100); \ + char** strings = backtrace_symbols(array, size); \ + if (strings != NULL) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames", size); \ + for (int32_t i = 0; i < size; i++) { \ + taosPrintLog(flags, level, dflag, "frame:%d, %s", i, strings[i]); \ + } \ + } \ + \ + taosMemoryFree(strings); \ + } +#else +#define taosPrintTrace(flags, level, dflag) \ + { \ + taosPrintLog(flags, level, dflag, \ + "backtrace not implemented on windows, so detailed stack information cannot be printed"); \ + } +#endif + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a6155f6611f6aa0acb57a56625f3d38c98bb9bc2..7ac9b6e1fd0c36cb9bbad04b1084947046aa96ce 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -40,61 +40,61 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc -// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //2.x -// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //2.x -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) -// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //2.x -// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //2.x -// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) //2.x -// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //2.x -// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //2.x -// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //2.x +// #define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) // 2.x +// #define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) // 2.x +// #define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) // 2.x +// #define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) // 2.x +// #define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) // 2.x +// #define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) // 2.x +// #define TSDB_CODE_RPC_LAST_SESSION_NOT_FINI. TAOS_DEF_ERROR_CODE(0, 0x0007) // 2.x +// #define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) // 2.x +// #define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) // 2.x +// #define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) // 2.x #define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) -// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //2.x -// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //2.x -// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //2.x -// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //2.x -// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //2.x -// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //2.x -// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //2.x -#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) +// #define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) // 2.x +// #define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) // 2.x +// #define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) // 2.x +// #define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) // 2.x +// #define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) // 2.x +// #define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) // 2.x +// #define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) // 2.x +#define TSDB_CODE_TIME_UNSYNCED TAOS_DEF_ERROR_CODE(0, 0x0013) // +// #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) // 2.x #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) -// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //2.x -#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) -#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) -#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) +// #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) // 2.x +#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017) // +#define TSDB_CODE_RPC_BROKEN_LINK TAOS_DEF_ERROR_CODE(0, 0x0018) // +#define TSDB_CODE_RPC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0019) // //common & util -#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) -#define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) +#define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) // +// #define TSDB_CODE_MEMORY_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0101) // 2.x #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0102) // #define TSDB_CODE_COM_INVALID_CFG_MSG TAOS_DEF_ERROR_CODE(0, 0x0103) // 2.x -#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) -#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) -#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) -#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) -#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) -#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) -#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) - -#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) -#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) -#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) -#define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) -#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) -#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) +#define TSDB_CODE_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0104) // +// #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0105) // 2.x +#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0106) // internal +#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0107) // internal +#define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0108) // internal +#define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0109) // internal +#define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x010A) // internal + +#define TSDB_CODE_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0110) // +#define TSDB_CODE_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0111) // internal +#define TSDB_CODE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0112) // +// #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x0113) +// #define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x0114) +#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0115) // #define TSDB_CODE_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0116) // -#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) -#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) -#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) -#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) -#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) -#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) -#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) -#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) -#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) +#define TSDB_CODE_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x0117) // internal +#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x0118) // +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x0119) // internal +#define TSDB_CODE_INVALID_OPTION TAOS_DEF_ERROR_CODE(0, 0x011A) // internal +#define TSDB_CODE_INVALID_JSON_FORMAT TAOS_DEF_ERROR_CODE(0, 0x011B) // internal +#define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x011C) // internal +#define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x011D) // internal +#define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x011E) // internal +#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // @@ -126,11 +126,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) #define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) #define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) -#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) +// #define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) // 2.x #define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) #define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) #define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) -#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) +// #define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) // 2.x #define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) #define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) #define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) @@ -156,34 +156,44 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) // mnode-common +// #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x +// #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) // 2.x +// #define TSDB_CODE_MND_ACTION_NEED_REPROCESSEDTAOS_DEF_ERROR_CODE(0, 0x0302) // 2.x #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) -#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) -#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) +// #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0304) // 2.x +// #define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0305) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0306) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0307) // 2.x +// #define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0308) // 2.x +// #define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x0309) // 2.x +// #define TSDB_CODE_MND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x030A) // 2.x #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x030B) #define TSDB_CODE_MND_INVALID_QUERY_ID TAOS_DEF_ERROR_CODE(0, 0x030C) -#define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) +// #define TSDB_CODE_MND_INVALID_STREAM_ID TAOS_DEF_ERROR_CODE(0, 0x030D) // 2.x #define TSDB_CODE_MND_INVALID_CONN_ID TAOS_DEF_ERROR_CODE(0, 0x030E) -#define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) -#define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) -#define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) -#define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) -#define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) +// #define TSDB_CODE_MND_MNODE_IS_RUNNING TAOS_DEF_ERROR_CODE(0, 0x0310) // 2.x +// #define TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC TAOS_DEF_ERROR_CODE(0, 0x0311) // 2.x +// #define TSDB_CODE_MND_FAILED_TO_START_SYNC TAOS_DEF_ERROR_CODE(0, 0x0312) // 2.x +// #define TSDB_CODE_MND_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0313) // 2.x +// #define TSDB_CODE_MND_FAILED_TO_INIT_STEP TAOS_DEF_ERROR_CODE(0, 0x0314) // 2.x #define TSDB_CODE_MND_USER_DISABLED TAOS_DEF_ERROR_CODE(0, 0x0315) // mnode-sdb -#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) -#define TSDB_CODE_SDB_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // internal -#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) +#define TSDB_CODE_SDB_OBJ_ALREADY_THERE TAOS_DEF_ERROR_CODE(0, 0x0320) // internal +// #define TSDB_CODE_MND_SDB_ERROR TAOS_DEF_ERROR_CODE(0, 0x0321) // 2.x +#define TSDB_CODE_SDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0322) // internal #define TSDB_CODE_SDB_OBJ_NOT_THERE TAOS_DEF_ERROR_CODE(0, 0x0323) -#define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) -#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0326) -#define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0327) -#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0328) -#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x0329) +// #define TSDB_CODE_MND_SDB_INVAID_META_ROW TAOS_DEF_ERROR_CODE(0, 0x0324) // 2.x +// #define TSDB_CODE_SDB_INVALID_KEY_TYPE TAOS_DEF_ERROR_CODE(0, 0x0325) // 2.x +#define TSDB_CODE_SDB_INVALID_ACTION_TYPE TAOS_DEF_ERROR_CODE(0, 0x0326) // internal +// #define TSDB_CODE_SDB_INVALID_STATUS_TYPE TAOS_DEF_ERROR_CODE(0, 0x0327) // unused +#define TSDB_CODE_SDB_INVALID_DATA_VER TAOS_DEF_ERROR_CODE(0, 0x0328) // internal +#define TSDB_CODE_SDB_INVALID_DATA_LEN TAOS_DEF_ERROR_CODE(0, 0x0329) // internal #define TSDB_CODE_SDB_INVALID_DATA_CONTENT TAOS_DEF_ERROR_CODE(0, 0x032A) -#define TSDB_CODE_SDB_INVALID_WAl_VER TAOS_DEF_ERROR_CODE(0, 0x032B) +// #define TSDB_CODE_SDB_INVALID_WAl_VER TAOS_DEF_ERROR_CODE(0, 0x032B) // unused #define TSDB_CODE_SDB_OBJ_CREATING TAOS_DEF_ERROR_CODE(0, 0x032C) #define TSDB_CODE_SDB_OBJ_DROPPING TAOS_DEF_ERROR_CODE(0, 0x032D) @@ -217,22 +227,31 @@ int32_t* taosGetErrno(); // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) +// #define TSDB_CODE_MND_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0361) // 2.x #define TSDB_CODE_MND_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0362) +// #define TSDB_CODE_MND_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0363) // 2.x #define TSDB_CODE_MND_TOO_MANY_TAGS TAOS_DEF_ERROR_CODE(0, 0x0364) #define TSDB_CODE_MND_TOO_MANY_COLUMNS TAOS_DEF_ERROR_CODE(0, 0x0365) +// #define TSDB_CODE_MND_TOO_MANY_TIMESERIES TAOS_DEF_ERROR_CODE(0, 0x0366) // 2.x +// #define TSDB_CODE_MND_NOT_SUPER_TABLE TAOS_DEF_ERROR_CODE(0, 0x0367) // 2.x +// #define TSDB_CODE_MND_COL_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x0368) // 2.x #define TSDB_CODE_MND_TAG_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0369) #define TSDB_CODE_MND_TAG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036A) #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x036B) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x036C) +// #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) // mnode-func #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) +// #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0374) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) +// #define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) // 2.x +// #define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) // 2.x #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) @@ -298,6 +317,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_CONFLICT TAOS_DEF_ERROR_CODE(0, 0x03D3) #define TSDB_CODE_MND_TRANS_CLOG_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x03D4) #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) +#define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq @@ -323,6 +343,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) #define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) #define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4) +#define TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB TAOS_DEF_ERROR_CODE(0, 0x03F5) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) @@ -330,13 +351,49 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482) // dnode -#define TSDB_CODE_NODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408) -#define TSDB_CODE_NODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409) -#define TSDB_CODE_NODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040A) +// #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x +// #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) // 2.x +// #define TSDB_CODE_DND_NO_WRITE_ACCESS TAOS_DEF_ERROR_CODE(0, 0x0402) // 2.x +// #define TSDB_CODE_DND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0403) // 2.x +// #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0404) // 2.x +// #define TSDB_CODE_DND_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0405) // 2.x +// #define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0406) // 2.x +// #define TSDB_CODE_DND_VNODE_OPEN_FAILED TAOS_DEF_ERROR_CODE(0, 0x0407) // 2.x +#define TSDB_CODE_DNODE_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0408) +#define TSDB_CODE_MNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0409) +#define TSDB_CODE_MNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040A) +#define TSDB_CODE_MNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040B) +#define TSDB_CODE_QNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040C) +#define TSDB_CODE_QNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x040D) +#define TSDB_CODE_QNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040E) +#define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F) +#define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410) +#define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) // vnode +// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x +// #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x +// #define TSDB_CODE_VND_ACTION_NEED_REPROCESS. TAOS_DEF_ERROR_CODE(0, 0x0502) // 2.x #define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) +// #define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) // 2.x +// #define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) // 2.x +// #define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) // 2.x +// #define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) // 2.x +// #define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) // 2.x +// #define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) // 2.x +// #define TSDB_CODE_VND_INVALID_VRESION_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) // 2.x +// #define TSDB_CODE_VND_IS_FULL TAOS_DEF_ERROR_CODE(0, 0x050B) // 2.x +// #define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) // 2.x +// #define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) // 2.x +// #define TSDB_CODE_VND_IS_BALANCING TAOS_DEF_ERROR_CODE(0, 0x050E) // 2.x +// #define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) // 2.x +// #define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) // 2.x #define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) +// #define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) // 2.x +// #define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) // 2.x +// #define TSDB_CODE_WAIT_THREAD_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0515) // 2.x +#define TSDB_CODE_VND_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0520) // internal +#define TSDB_CODE_VND_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0521) // internal #define TSDB_CODE_VND_HASH_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0522) #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0524) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0525) @@ -344,6 +401,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_COL_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x0527) #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) +#define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -353,8 +411,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) #define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) #define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) +// #define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) // 2.x +// #define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) // 2.x #define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) #define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) #define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) @@ -368,6 +426,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) #define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) +// #define TSDB_CODE_TDB_INCOMPLETE_DFILESET TAOS_DEF_ERROR_CODE(0, 0x0617) // 2.x #define TSDB_CODE_TDB_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0618) #define TSDB_CODE_TDB_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0619) #define TSDB_CODE_TDB_STB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x061A) @@ -378,8 +437,9 @@ int32_t* taosGetErrno(); // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) #define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) -#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) -#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) +// #define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) // 2.x +// #define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) // 2.x +// #define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) // 2.x #define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) #define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) #define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) @@ -391,6 +451,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070E) #define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) +// #define TSDB_CODE_QRY_INVALID_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0710) // 2.x +// #define TSDB_CODE_QRY_RESULT_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0711) // 2.x #define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0720) #define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0721) #define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0722) @@ -426,7 +488,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D) // sync +// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x +// #define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) // 2.x +// #define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) // 2.x #define TSDB_CODE_SYN_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x0903) +// #define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) // 2.x +// #define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) // 2.x +// #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x +// #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x #define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) #define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) @@ -443,8 +515,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) #define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) -#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) -#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) +// #define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) +// #define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) #define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) #define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) #define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) @@ -455,16 +527,17 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TQ_NO_COMMITTED_OFFSET TAOS_DEF_ERROR_CODE(0, 0x0A0D) // wal -#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) +// #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) // 2.x #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) -#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) +// #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // 2.x #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) #define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) #define TSDB_CODE_WAL_LOG_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x1007) // tfs +// #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) // 2.x #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) #define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) #define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) @@ -473,7 +546,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) #define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) #define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) -#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) +// #define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x220F) // 2.x // catalog #define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) @@ -628,7 +701,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) #define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152) -#define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153) +// #define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153) #define TSDB_CODE_RSMA_REMOVE_EXISTS TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x3155) #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 38eb0d8fc6691459acfe75148b16eb3858123961..b1bd09e1238d55aa0505f9d496dca4a66f191962 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -17,6 +17,7 @@ #define _TD_UTIL_CODING_H_ #include "os.h" +#include "tlog.h" #ifdef __cplusplus extern "C" { diff --git a/include/util/tdef.h b/include/util/tdef.h index 6ce7b737a884337e912c3d1d8a161347ebdd8ac8..7cd4f57771a51329d36e023f4f0d1176447261e6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -189,12 +189,13 @@ typedef enum ELogicConditionType { #define TSDB_MAX_COLUMNS 4096 #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -280,7 +281,7 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_ROLE_MGMT 1 #define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_REPLICA 5 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 #define TSDB_TBNAME_COLUMN_INDEX (-1) @@ -307,8 +308,9 @@ typedef enum ELogicConditionType { #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute #define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) -#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute -#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute +#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. +#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved. #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MAX_MINROWS_FBLOCK 1000 @@ -336,7 +338,7 @@ typedef enum ELogicConditionType { #define TSDB_DB_STRICT_ON_STR "on" #define TSDB_DB_STRICT_OFF 0 #define TSDB_DB_STRICT_ON 1 -#define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_OFF +#define TSDB_DEFAULT_DB_STRICT TSDB_DB_STRICT_ON #define TSDB_CACHE_MODEL_STR_LEN sizeof(TSDB_CACHE_MODEL_LAST_VALUE_STR) #define TSDB_CACHE_MODEL_NONE_STR "none" #define TSDB_CACHE_MODEL_LAST_ROW_STR "last_row" @@ -381,13 +383,16 @@ typedef enum ELogicConditionType { #define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0 #define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0 -#define TSDB_MIN_ROLLUP_MAX_DELAY 1 // unit millisecond -#define TSDB_MAX_ROLLUP_MAX_DELAY (15 * 60 * 1000) -#define TSDB_MIN_ROLLUP_WATERMARK 0 // unit millisecond -#define TSDB_MAX_ROLLUP_WATERMARK (15 * 60 * 1000) -#define TSDB_DEFAULT_ROLLUP_WATERMARK 5000 -#define TSDB_MIN_TABLE_TTL 0 -#define TSDB_DEFAULT_TABLE_TTL 0 +#define TSDB_MIN_ROLLUP_MAX_DELAY 1 // unit millisecond +#define TSDB_MAX_ROLLUP_MAX_DELAY (15 * 60 * 1000) +#define TSDB_MIN_ROLLUP_WATERMARK 0 // unit millisecond +#define TSDB_MAX_ROLLUP_WATERMARK (15 * 60 * 1000) +#define TSDB_DEFAULT_ROLLUP_WATERMARK 5000 +#define TSDB_MIN_ROLLUP_DELETE_MARK 0 // unit millisecond +#define TSDB_MAX_ROLLUP_DELETE_MARK INT64_MAX +#define TSDB_DEFAULT_ROLLUP_DELETE_MARK 900000 // 900s +#define TSDB_MIN_TABLE_TTL 0 +#define TSDB_DEFAULT_TABLE_TTL 0 #define TSDB_MIN_EXPLAIN_RATIO 0 #define TSDB_MAX_EXPLAIN_RATIO 1 @@ -493,6 +498,9 @@ enum { // sort page size by default #define DEFAULT_PAGESIZE 4096 +#define VNODE_TIMEOUT_SEC 60 +#define MNODE_TIMEOUT_SEC 10 + #ifdef __cplusplus } #endif diff --git a/include/util/tlog.h b/include/util/tlog.h index 68b004cda74f158791d38decf73346425bf44de9..e6ef7f388ff7af4bab6c29a8a62d39d01d31f256 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -38,6 +38,7 @@ typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); extern bool tsLogEmbedded; extern bool tsAsyncLog; +extern bool tsAssert; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern LogFp tsLogFp; @@ -82,6 +83,10 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...); +#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__) +#define ASSERT(condition) ASSERTS(condition, "assert info not provided") + // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 8b46bbd0642366d9dce26f406939b3c3112cb97e..1f6b205cdf8a2faa045a7a08b9008cb271a741a1 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -65,6 +65,7 @@ typedef struct STaosQnode { STaosQnode *next; STaosQueue *queue; int64_t timestamp; + int64_t dataSize; int32_t size; int8_t itype; int8_t reserved[3]; @@ -103,7 +104,7 @@ typedef struct STaosQall { STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void *taosAllocateQitem(int32_t size, EQItype itype); +void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize); void taosFreeQitem(void *pItem); void taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 4e32e9ad8faafa1b9f1df923c7386001d9a5f70d..edec338c55c4ada6a7669a670a89d5db68e97ca3 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -187,7 +187,7 @@ if [[ $productName == "TDengine" ]]; then git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet rm -rf ${install_dir}/connector/dotnet/.git ||: # cp -r ${connector_dir}/nodejs ${install_dir}/connector - git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + git clone --depth 1 https://github.com/taosdata/taos-connector-rust ${install_dir}/connector/rust rm -rf ${install_dir}/connector/rust/.git ||: fi fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 2776683a249c3376b9cf5e5d54ba43f534f3df47..4169bed3eb3b4f732ef928425e92c7ff7ee20561 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -318,7 +318,7 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then git clone --depth 1 https://github.com/taosdata/taos-connector-dotnet ${install_dir}/connector/dotnet rm -rf ${install_dir}/connector/dotnet/.git ||: - git clone --depth 1 https://github.com/taosdata/libtaos-rs ${install_dir}/connector/rust + git clone --depth 1 https://github.com/taosdata/taos-connector-rust ${install_dir}/connector/rust rm -rf ${install_dir}/connector/rust/.git ||: # cp -r ${connector_dir}/python ${install_dir}/connector @@ -379,4 +379,4 @@ if [ -n "${taostools_bin_files}" ]; then fi fi -cd ${curr_dir} \ No newline at end of file +cd ${curr_dir} diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index aae20c587d2b2d6373126ba6fcbbce8eb2daed14..c3b79d78298af72ce3e9b6894c7f66ef048a2382 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,7 +149,6 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; typedef struct SResultColumn { @@ -171,9 +170,9 @@ typedef struct SReqResultInfo { char** convertBuf; TAOS_ROW row; SResultColumn* pCol; - uint32_t numOfRows; + uint64_t numOfRows; // from int32_t change to int64_t uint64_t totalRows; - uint32_t current; + uint64_t current; bool localResultFetched; bool completed; int32_t precision; diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61db570f2c6fb488d2964fc38c2cfd7f4..c29f495201d7b9ec0360abf6a5414798739b3da8 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h new file mode 100644 index 0000000000000000000000000000000000000000..b74d0b34943af7e5a61e3c0fd8d4bc481736c9d6 --- /dev/null +++ b/source/client/inc/clientSml.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "clientInt.h" +#include "osThread.h" +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tcommon.h" +#include "tdef.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmsg.h" +#include "tname.h" +#include "ttime.h" +#include "ttypes.h" +#include "cJSON.h" + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ + if (unlikely(*sql == SPACE)) \ + sql++; \ + else \ + break; \ + } + +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + +#define TS "_ts" +#define TS_LEN 3 +#define VALUE "_value" +#define VALUE_LEN 6 + +#define MAX_RETRY_TIMES 5 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_NULL, + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; + + SArray *colArray; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + int32_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; + void *key; // for openTsdb + + SArray *tags; + + // elements are SHashObj for find by key quickly + SArray *cols; + STableDataCxt *tableDataCtx; +} SSmlTableInfo; + +typedef struct { + SArray *tags; // save the origin order to create table + SHashObj *tagHash; // elements are + + SArray *cols; + SHashObj *colHash; + + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + int32_t numOfAlterColSTables; + int32_t numOfAlterTagSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + +typedef struct { + int64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool reRun; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; + + NodeList *childTables; + NodeList *superTables; + SHashObj *pVgHash; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + SSmlCostInfo cost; + int32_t lineNum; + SSmlMsgBuf msgBuf; + +// cJSON *root; // for parse json + int8_t offset[4]; + SSmlLineInfo *lines; // element is SSmlLineInfo + + // + SArray *preLineTagKV; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; +} SSmlHandle; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (preKV->keyLen == kv.keyLen && memcmp(preKV->key, kv.key, kv.keyLen) == 0) + +extern int64_t smlFactorNS[3]; +extern int64_t smlFactorS[3]; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +SSmlHandle *smlBuildSmlInfo(TAOS *taos); +void smlDestroyInfo(SSmlHandle *info); +void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); +void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); +SArray *smlJsonParseTags(char *start, char *end); +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +int nodeListSize(NodeList* list); +bool smlDoubleToInt64OverFlow(double num); +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); +int8_t smlGetTsTypeByLen(int32_t len); +SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); +SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlSetCTableName(SSmlTableInfo *oneTable); +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t is_same_child_table_telnet(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); +int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseJSON(SSmlHandle *info, char *payload); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 1bda1684c4ec9da2697617e3707c2616c621c1aa..2b42de93e3f45ccb95c085c31825d4e0a117538c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -21,8 +21,6 @@ extern "C" { #endif #include "catalog.h" -typedef void STableDataCxt; - typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, @@ -71,10 +69,11 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { - int32_t affectedRows; - SRequestObj *pRequest; - SHashObj *pBlockHash; - bool autoCreateTbl; + int32_t affectedRows; + SRequestObj *pRequest; + SHashObj *pBlockHash; + STableDataCxt *pCurrBlock; + SSubmitTbData *pCurrTbData; } SStmtExecInfo; typedef struct SStmtSQLInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6f1414b72d8a3ede9752be1c4fa4594c547a7165..98c96dcfb1da0242453de3ee9e42b900ec6fafd6 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -76,13 +76,20 @@ static void deregisterRequest(SRequestObj *pRequest) { "current:%d, app current:%d", pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); + tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 + "us, exec:%" PRId64 "us, stmtType:%d", + duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, + pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - + pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd, + pRequest->stmtType); + if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { - // tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - // "us, exec:%" PRId64 "us", - // duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, - // pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - - // pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); - atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); +// tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 +// "us, exec:%" PRId64 "us", +// duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, +// pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - +// pRequest->metric.ctgEnd, pRequest->metric.execEnd - pRequest->metric.semanticEnd); +// atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { // tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 // "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%" PRIx64, @@ -146,8 +153,7 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; rpcInit.dfp = destroyAhandle; - rpcInit.retryLimit = tsRpcRetryLimit; - rpcInit.retryInterval = tsRpcRetryInterval; + rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryMaxInterval = tsRedirectMaxPeriod; @@ -231,10 +237,9 @@ void destroyTscObj(void *pObj) { tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); - int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - if (0 == connNum) { - destroyAppInst(pTscObj->pAppInfo); - } + // In any cases, we should not free app inst here. Or an race condition rises. + /*int64_t connNum = */ atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFree(pTscObj); @@ -244,14 +249,14 @@ void destroyTscObj(void *pObj) { void *createTscObj(const char *user, const char *auth, const char *db, int32_t connType, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)taosMemoryCalloc(1, sizeof(STscObj)); if (NULL == pObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pObj->pRequests = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (NULL == pObj->pRequests) { taosMemoryFree(pObj); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -266,7 +271,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); @@ -281,7 +285,7 @@ int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, ri void *createRequest(uint64_t connId, int32_t type, int64_t reqid) { SRequestObj *pRequest = (SRequestObj *)taosMemoryCalloc(1, sizeof(SRequestObj)); if (NULL == pRequest) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -397,8 +401,8 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { - tscInitRes = -1; - return; + // ignore create log failed, only print + printf(" WARING: Create taoslog failed. configDir=%s\n", configDir); } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { @@ -408,7 +412,9 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); - taosConvInit(); + if (taosConvInit() != 0) { + ASSERTS(0, "failed to init conv"); + } rpcInit(); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6bdc8352174840eea1ff9f1e5227ddcd787b2a64..47ed2cf035a13314b81f2208718d2c466efb5c68 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -69,7 +69,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog } else { SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == vgInfo) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -82,7 +82,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (NULL == vgInfo->vgHash) { taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } @@ -92,7 +92,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo->vgHash); taosMemoryFree(vgInfo); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _return; } } @@ -366,7 +366,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc)); if (NULL == desc.subDesc) { releaseRequest(*rid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } code = schedulerGetTasksStatus(pRequest->body.queryJob, desc.subDesc); @@ -394,14 +394,14 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (NULL == pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SQueryHbReqBasic *hbBasic = (SQueryHbReqBasic *)taosMemoryCalloc(1, sizeof(SQueryHbReqBasic)); if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); releaseTscObj(connKey->tscRid); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } hbBasic->connId = pTscObj->connId; @@ -419,7 +419,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { tscWarn("taosArrayInit %d queryDesc failed", numOfQueries); releaseTscObj(connKey->tscRid); taosMemoryFree(hbBasic); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); @@ -613,7 +613,7 @@ static FORCE_INLINE void hbMgrInitHandle() { SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); @@ -737,7 +737,7 @@ static void *hbThreadFunc(void *param) { int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosArrayPush(mgr, &pAppHbMgr); @@ -748,7 +748,7 @@ static void *hbThreadFunc(void *param) { SMsgSendInfo *pInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ca9f559f6b0825172b4db93c1410394482b13d9a..97ac315b77b48c43061dac6f0586ac5876234749 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -166,7 +166,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, tscError("0x%" PRIx64 " failed to prepare sql string buffer, %s", (*pRequest)->self, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } strntolower((*pRequest)->sqlstr, sql, (int32_t)sqlLen); @@ -179,7 +179,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, if (pParam == NULL) { destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } tsem_init(&pParam->sem, 0, 0); @@ -190,15 +190,16 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->body.param = param; STscObj* pTscObj = (*pRequest)->pTscObj; - if (taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, - sizeof((*pRequest)->self))) { + int32_t err = taosHashPut(pTscObj->pRequests, &(*pRequest)->self, sizeof((*pRequest)->self), &(*pRequest)->self, + sizeof((*pRequest)->self)); + if (err) { tscError("%" PRId64 " failed to add to request container, reqId:0x%" PRIx64 ", conn:%" PRId64 ", %s", (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); taosMemoryFree(param); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } (*pRequest)->allocatorRefId = -1; @@ -209,7 +210,7 @@ int32_t buildRequest(uint64_t connId, const char* sql, int sqlLen, void* param, (*pRequest)->self, (*pRequest)->requestId, pTscObj->id, sql); destroyRequest(*pRequest); *pRequest = NULL; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -232,7 +233,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, @@ -317,7 +317,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); } @@ -609,7 +609,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayDestroy(pDbVgList); @@ -670,7 +670,7 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* } default: tscError("unknown query policy: %d", tsQueryPolicy); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } _return: @@ -1111,7 +1111,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) { int32_t tblNum = taosArrayGetSize(pRequest->tableList); if (dbNum <= 0 && tblNum <= 0) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); @@ -1206,7 +1206,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t SAppInstInfo* pAppInfo, int connType) { STscObj* pTscObj = createTscObj(user, auth, db, connType, pAppInfo); if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return pTscObj; } @@ -1243,7 +1243,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1502,7 +1502,7 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) return NULL; } - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResInfo->numOfRows, pResInfo->totalRows, pResInfo->completed, pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; @@ -1620,7 +1620,8 @@ int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1916,7 +1917,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pData = (void*)pRsp->data; - pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->numOfRows = htobe64(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); pResultInfo->payloadLen = htonl(pRsp->compLen); @@ -1946,8 +1947,6 @@ TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* de rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.compressSize = tsCompressMsgSize; rpcInit.user = "_dnd"; - rpcInit.retryLimit = tsRpcRetryLimit; - rpcInit.retryInterval = tsRpcRetryInterval; clientRpc = rpcOpen(&rpcInit); if (clientRpc == NULL) { @@ -2269,14 +2268,14 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { taosAsyncQueryImpl(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly); tsem_wait(¶m->sem); - SRequestObj *pRequest = NULL; + SRequestObj* pRequest = NULL; if (param->pRequest != NULL) { param->pRequest->syncQuery = true; pRequest = param->pRequest; } else { taosMemoryFree(param); } - + return pRequest; } @@ -2292,13 +2291,13 @@ TAOS_RES* taosQueryImplWithReqid(TAOS* taos, const char* sql, bool validateOnly, taosAsyncQueryImplWithReqid(*(int64_t*)taos, sql, syncQueryFn, param, validateOnly, reqid); tsem_wait(¶m->sem); - SRequestObj *pRequest = NULL; + SRequestObj* pRequest = NULL; if (param->pRequest != NULL) { param->pRequest->syncQuery = true; pRequest = param->pRequest; } else { taosMemoryFree(param); } - + return pRequest; } diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index b5a6ebadeec74e6565fd5225144cd139f6c9aaab..34fc480432f5dac71f80b6316718fe22397eb125 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -136,7 +136,7 @@ int32_t check_for_params(jobject jobj, jlong conn, jlong res) { } if ((TAOS_RES *)res == NULL) { - jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)conn); + jniError("jobj:%p, conn:%p, param res is null", jobj, (TAOS *)conn); return JNI_RESULT_SET_NULL; } @@ -393,9 +393,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp( return code; } - taos_free_result((void *)res); jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res); - + taos_free_result((void *)res); return JNI_SUCCESS; } @@ -489,7 +488,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn numOfFields); return JNI_FETCH_END; } else { - jniDebug("jobj:%p, conn:%p, interrupted query", jobj, tscon); + jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code, taos_errstr(result)); return JNI_RESULT_SET_NULL; } } @@ -584,7 +583,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon); + jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 976d1dd1b0aef2364b5d2dd05ddc15b60e19d049..896d1d6ca6f34b06e359334eabaf4eb53434b1d5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -21,12 +21,12 @@ #include "os.h" #include "query.h" #include "scheduler.h" +#include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "version.h" -#include "tdatablock.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -146,7 +146,6 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return terrno; } @@ -154,12 +153,11 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_QRY_NOT_READY : ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { - if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_QRY_NOT_READY; return (const char *)tstrerror(terrno); } @@ -171,8 +169,7 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_QRY_NOT_READY) - : (const char *)tstrerror(pRequest->code); + return (const char *)tstrerror(pRequest->code); } } @@ -181,16 +178,18 @@ void taos_free_result(TAOS_RES *res) { return; } + tscDebug("taos free res %p", res); + if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; tscDebug("0x%" PRIx64 " taos_free_result start to free query", pRequest->requestId); destroyRequest(pRequest); } else if (TD_RES_TMQ_METADATA(res)) { SMqTaosxRspObj *pRsp = (SMqTaosxRspObj *)res; - if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); - if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); - if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); + taosArrayDestroy(pRsp->rsp.blockDataLen); + taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); + taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); // taosx taosArrayDestroy(pRsp->rsp.createTableLen); taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree); @@ -200,10 +199,10 @@ void taos_free_result(TAOS_RES *res) { taosMemoryFree(pRsp); } else if (TD_RES_TMQ(res)) { SMqRspObj *pRsp = (SMqRspObj *)res; - if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); - if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); - if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); + taosArrayDestroy(pRsp->rsp.blockDataLen); + taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); + taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); taosMemoryFree(pRsp); @@ -438,11 +437,23 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } +// return int32_t int taos_affected_rows(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { return 0; } + SRequestObj *pRequest = (SRequestObj *)res; + SReqResultInfo *pResInfo = &pRequest->body.resInfo; + return (int)pResInfo->numOfRows; +} + +// return int64_t +int64_t taos_affected_rows64(TAOS_RES *res) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res)) { + return 0; + } + SRequestObj *pRequest = (SRequestObj *)res; SReqResultInfo *pResInfo = &pRequest->body.resInfo; return pResInfo->numOfRows; @@ -787,9 +798,11 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgEnd = taosGetTimestampUs(); - qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); + qDebug("0x%" PRIx64 " start to continue parse, reqId:0x%" PRIx64 ", code:%s", pRequest->self, pRequest->requestId, + tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { + //pWrapper->pCatalogReq->forceUpdate = false; code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); } @@ -853,7 +866,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, @@ -870,6 +882,11 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (pRequest->retry++ > REQUEST_TOTAL_EXEC_TIMES) { code = pRequest->prevCode; + terrno = code; + pRequest->code = code; + tscDebug("call sync query cb with code: %s", tstrerror(code)); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); + return; } if (TSDB_CODE_SUCCESS == code) { @@ -920,6 +937,17 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; + + if (NEED_CLIENT_HANDLE_ERROR(code)) { + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, + pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); + pRequest->prevCode = code; + doAsyncQuery(pRequest, true); + return; + } + terrno = code; pRequest->code = code; pRequest->body.queryFp(pRequest->body.param, pRequest, code); @@ -959,7 +987,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - tscDebug("0x%" PRIx64 " fetch results, numOfRows:%d total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, + tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, pRequest->self, pResultInfo->numOfRows, pResultInfo->totalRows, pResultInfo->completed, pRequest->requestId); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4bd74a842fcb5d758d66aa4c6949728d12e4180c..85027ff371c921cc6bf9137d4f7eaece092c2e7d 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -86,7 +86,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { - setErrno(pRequest, TSDB_CODE_MND_APP_ERROR); + setErrno(pRequest, TSDB_CODE_APP_ERROR); tsem_post(&pRequest->body.rspSem); goto End; } @@ -450,7 +450,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 3242926183684b4b20ff12d41f77d61d887d4dc7..8c48a745f542dce0140f0d99690b14cecba5460e 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1378,7 +1378,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1569,7 +1569,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } dst->vg = vgData; @@ -1696,7 +1696,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } } - + code = smlBuildOutput(pQuery, pVgHash); if (code != TSDB_CODE_SUCCESS) { uError("smlBuildOutput failed"); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 163e7044243195ee9d36c14e5229bd75f291d67b..1fb74cb1286ab47d12c286492a633d7b11aba046 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,218 +1,99 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include #include -#include "cJSON.h" -#include "catalog.h" -#include "clientInt.h" -#include "osSemaphore.h" -#include "osThread.h" -#include "query.h" -#include "taos.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tdef.h" -#include "tglobal.h" -#include "tlog.h" -#include "tmsg.h" -#include "tname.h" -#include "ttime.h" -#include "ttypes.h" - -//================================================================================================= - -#define SPACE ' ' -#define COMMA ',' -#define EQUAL '=' -#define QUOTE '"' -#define SLASH '\\' - -#define JUMP_SPACE(sql, sqlEnd) \ - while (sql < sqlEnd) { \ - if (*sql == SPACE) \ - sql++; \ - else \ - break; \ - } -// comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) - -#define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ - keyLen--; \ - } \ - } - -#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) -#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) - -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define TS "_ts" -#define TS_LEN 3 -#define VALUE "_value" -#define VALUE_LEN 6 - -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " - -#define MAX_RETRY_TIMES 5 -//================================================================================================= -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef enum { - SCHEMA_ACTION_NULL, - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} SSmlLineInfo; - -typedef struct { - const char *sTableName; // super table name - int32_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly - SArray *cols; -} SSmlTableInfo; - -typedef struct { - SArray *tags; // save the origin order to create table - SHashObj *tagHash; // elements are - - SArray *cols; - SHashObj *colHash; - - STableMeta *tableMeta; -} SSmlSTableMeta; - -typedef struct { - int32_t len; - char *buf; -} SSmlMsgBuf; - -typedef struct { - int32_t code; - int32_t lineNum; - - int32_t numOfSTables; - int32_t numOfCTables; - int32_t numOfCreateSTables; - int32_t numOfAlterColSTables; - int32_t numOfAlterTagSTables; - - int64_t parseTime; - int64_t schemaTime; - int64_t insertBindTime; - int64_t insertRpcTime; - int64_t endTime; -} SSmlCostInfo; - -typedef struct { - SRequestObj *request; - tsem_t sem; - int32_t cnt; - int32_t total; - TdThreadSpinlock lock; -} Params; - -typedef struct { - int64_t id; - Params *params; - - SMLProtocolType protocol; - int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; - int32_t ttl; - - SHashObj *childTables; - SHashObj *superTables; - SHashObj *pVgHash; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - SSmlCostInfo cost; - int32_t affectedRows; - SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false - - cJSON *root; // for parse json -} SSmlHandle; -//================================================================================================= - -//================================================================================================= -static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGenId() { - int64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); +#include "clientSml.h" + +int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; + +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn){ + NodeList *tmp = list; + while(tmp){ + if(fn == NULL){ + if(tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + }else{ + if(tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } - return id; + tmp = tmp->next; + } + return NULL; } -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn){ + NodeList *tmp = *list; + while (tmp){ + if(!tmp->data.used) break; + if(fn == NULL){ + if(tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + }else{ + if(tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if(tmp){ + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + }else{ + NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); + if(newNode == NULL){ + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; } -static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) { - void *val = taosHashGet(pHash, key, keyLen); - if (val) { - return true; +int nodeListSize(NodeList* list){ + int cnt = 0; + while(list){ + if(list->data.used) cnt++; + else break; + list = list->next; } - taosHashPut(pHash, key, keyLen, key, 1); + return cnt; +} + +inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } -static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); @@ -225,6 +106,438 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } + +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { + char *endPtr = NULL; + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); + if (unlikely(value + len != endPtr)) { + return -1; + } + + if(unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)){ + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if(unit > INT64_MAX / tsInt64){ + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; + } + + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); +} + +int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; + } +} + +SSmlTableInfo *smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; + } + + tag->sTableName = measure; + tag->sTableNameLen = measureLen; + + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; + } + +// tag->tags = taosArrayInit(16, sizeof(SSmlKv)); +// if (tag->tags == NULL) { +// uError("SML:smlBuildTableInfo failed to allocate memory"); +// goto cleanup; +// } + return tag; + + cleanup: + taosMemoryFree(tag); + return NULL; +} + +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for(int i = 0; i < taosArrayGetSize(tags); i++){ + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlSetCTableName(SSmlTableInfo *oneTable){ + smlParseTableName(oneTable->tags, oneTable->childTableName); + + if (strlen(oneTable->childTableName) == 0) { + SArray* dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, + oneTable->childTableName, 0}; + + buildChildTableName(&rName); + taosArrayDestroy(dst); + oneTable->uid = rName.uid; + } else { + oneTable->uid = *(uint64_t *)(oneTable->childTableName); + } + return TSDB_CODE_SUCCESS; +} + +SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; + } + + if(unlikely(!isDataFormat)){ + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + } + + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } + return meta; + + cleanup: + taosMemoryFree(meta); + return NULL; +} + +//uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +//} + +#define RETURN_FALSE \ +smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ +return false; + +#define SET_DOUBLE kvVal->type = TSDB_DATA_TYPE_DOUBLE;\ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) {\ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_FLOAT;\ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) {\ + errno = 0;\ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10);\ + if (errno == ERANGE) {\ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_BIGINT;\ + kvVal->i = tmp;\ + return true;\ + }\ + kvVal->type = TSDB_DATA_TYPE_BIGINT;\ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) {\ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_INT;\ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) {\ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_SMALLINT;\ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) {\ + errno = 0;\ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10);\ + if (errno == ERANGE || result < 0) {\ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ + kvVal->u = tmp;\ + return true;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UBIGINT;\ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UINT;\ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_USMALLINT;\ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_TINYINT;\ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) {\ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal);\ + return false;\ + }\ + kvVal->type = TSDB_DATA_TYPE_UTINYINT;\ + kvVal->u = result; + +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char * endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } + + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if(endptr[0] == 'f' || endptr[0] == 'F'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_DOUBLE + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_FLOAT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'i' || endptr[0] == 'I'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_BIGINT + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_INT + }else if(endptr[1] == '1' && endptr[2] == '6'){ + SET_SMALL_INT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'u' || endptr[0] == 'U'){ + if(endptr[1] == '6' && endptr[2] == '4'){ + SET_UBIGINT + }else if(endptr[1] == '3' && endptr[2] == '2'){ + SET_UINT + }else if(endptr[1] == '1' && endptr[2] == '6'){ + SET_USMALL_INT + }else{ + RETURN_FALSE + } + }else{ + RETURN_FALSE + } + } else if(left == 2){ + if(endptr[0] == 'i' || endptr[0] == 'I'){ + if(endptr[1] == '8') { + SET_TINYINT + }else{ + RETURN_FALSE + } + }else if(endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE + } + }else{ + RETURN_FALSE + } + } else if(left == 1){ + if(endptr[0] == 'i' || endptr[0] == 'I'){ + SET_BIGINT + }else if(endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + }else{ + RETURN_FALSE + } + } else { + RETURN_FALSE; + } + return true; +} + +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + + int32_t left = len - (endptr - pVal); + if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { + kvVal->type = TSDB_DATA_TYPE_DOUBLE; + kvVal->d = result; + } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { + if (!IS_VALID_FLOAT(result)) { + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_FLOAT; + kvVal->f = (float)result; + } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { + if (smlDoubleToInt64OverFlow(result)) { + errno = 0; + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); + if (errno == ERANGE) { + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_BIGINT; + kvVal->i = (int64_t)result; + } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { + if (result >= (double)UINT64_MAX || result < 0) { + errno = 0; + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); + if (errno == ERANGE || result < 0) { + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = tmp; + return true; + } + kvVal->type = TSDB_DATA_TYPE_UBIGINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { + if (!IS_VALID_INT(result)) { + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_INT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { + if (!IS_VALID_UINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UINT; + kvVal->u = result; + } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { + if (!IS_VALID_SMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_SMALLINT; + kvVal->i = result; + } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { + if (!IS_VALID_USMALLINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_USMALLINT; + kvVal->u = result; + } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { + if (!IS_VALID_TINYINT(result)) { + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_TINYINT; + kvVal->i = result; + } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { + if (!IS_VALID_UTINYINT(result)) { + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); + return false; + } + kvVal->type = TSDB_DATA_TYPE_UTINYINT; + kvVal->u = result; + } else { + smlBuildInvalidDataMsg(msg, "invalid data", pVal); + return false; + } + return true; +} + +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen){ + STableMeta *pTableMeta = NULL; + + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); + + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); + + catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + return pTableMeta; +} + + +static int64_t smlGenId() { + static volatile int64_t linesSmlHandleId = 0; + + int64_t id = 0; + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); + + return id; +} + static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, ESchemaAction *action, SSmlHandle *info) { uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; @@ -279,7 +592,7 @@ static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SH int32_t code = TSDB_CODE_SUCCESS; for (int j = 0; j < taosArrayGetSize(cols); ++j) { if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); if (code != TSDB_CODE_SUCCESS) { return code; @@ -301,7 +614,7 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool i = 1; } for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { taosHashCleanup(hashTmp); return -1; @@ -322,7 +635,7 @@ static int32_t getBytes(uint8_t type, int32_t length) { static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, SArray *results, int32_t numOfCols, bool isTag) { for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); ESchemaAction action = SCHEMA_ACTION_NULL; smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { @@ -423,13 +736,16 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); -end: + end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); return code; } static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if(info->dataFormat && !info->needModifySchema){ + return TSDB_CODE_SUCCESS; + } int32_t code = 0; SHashObj *hashTmp = NULL; STableMeta *pTableMeta = NULL; @@ -443,13 +759,13 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { conn.requestObjRefId = info->pRequest->self; conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; + NodeList *tmp = info->superTables; + while (tmp) { + SSmlSTableMeta *sTableData = (SSmlSTableMeta *)tmp->data.value; bool needCheckMeta = false; // for multi thread - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); + size_t superTableLen = (size_t)tmp->data.keyLen; + const void *superTable = tmp->data.key; memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); memcpy(pName.tname, superTable, superTableLen); @@ -598,1702 +914,274 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { sTableData->tableMeta = pTableMeta; - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + tmp = tmp->next; } return 0; -end: + end: taosHashCleanup(hashTmp); taosMemoryFreeClear(pTableMeta); // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); return code; } -static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - char *endptr = NULL; - double result = taosStr2Double(pVal, &endptr); - if (pVal == endptr) { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - - int32_t left = len - (endptr - pVal); - if (left == 0 || (left == 3 && strncasecmp(endptr, "f64", left) == 0)) { - kvVal->type = TSDB_DATA_TYPE_DOUBLE; - kvVal->d = result; - } else if ((left == 3 && strncasecmp(endptr, "f32", left) == 0)) { - if (!IS_VALID_FLOAT(result)) { - smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_FLOAT; - kvVal->f = (float)result; - } else if ((left == 1 && *endptr == 'i') || (left == 3 && strncasecmp(endptr, "i64", left) == 0)) { - if (smlDoubleToInt64OverFlow(result)) { - errno = 0; - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); - if (errno == ERANGE) { - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_BIGINT; - kvVal->i = (int64_t)result; - } else if ((left == 1 && *endptr == 'u') || (left == 3 && strncasecmp(endptr, "u64", left) == 0)) { - if (result >= (double)UINT64_MAX || result < 0) { - errno = 0; - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); - if (errno == ERANGE || result < 0) { - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = tmp; - return true; - } - kvVal->type = TSDB_DATA_TYPE_UBIGINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i32", left) == 0) { - if (!IS_VALID_INT(result)) { - smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_INT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u32", left) == 0) { - if (!IS_VALID_UINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UINT; - kvVal->u = result; - } else if (left == 3 && strncasecmp(endptr, "i16", left) == 0) { - if (!IS_VALID_SMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_SMALLINT; - kvVal->i = result; - } else if (left == 3 && strncasecmp(endptr, "u16", left) == 0) { - if (!IS_VALID_USMALLINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_USMALLINT; - kvVal->u = result; - } else if (left == 2 && strncasecmp(endptr, "i8", left) == 0) { - if (!IS_VALID_TINYINT(result)) { - smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_TINYINT; - kvVal->i = result; - } else if (left == 2 && strncasecmp(endptr, "u8", left) == 0) { - if (!IS_VALID_UTINYINT(result)) { - smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); - return false; - } - kvVal->type = TSDB_DATA_TYPE_UTINYINT; - kvVal->u = result; - } else { - smlBuildInvalidDataMsg(msg, "invalid data", pVal); - return false; - } - return true; -} - -static bool smlParseBool(SSmlKv *kvVal) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { - kvVal->i = TSDB_TRUE; - return true; - } - - if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { - kvVal->i = TSDB_FALSE; - return true; - } - - if ((len == 4) && !strncasecmp(pVal, "true", len)) { - kvVal->i = TSDB_TRUE; - return true; - } - if ((len == 5) && !strncasecmp(pVal, "false", len)) { - kvVal->i = TSDB_FALSE; - return true; - } - return false; -} - -static bool smlIsBinary(const char *pVal, uint16_t len) { - // binary: "abc" - if (len < 2) { - return false; - } - if (pVal[0] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} - -static bool smlIsNchar(const char *pVal, uint16_t len) { - // nchar: L"abc" - if (len < 3) { - return false; - } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} - -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { - char *endPtr = NULL; - int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { - return -1; - } - - return tsInt64; -} - -static int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; - } else { - return -1; - } -} - -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; - } -} - -static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); - } - - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; - } - - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return -1; - } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); - } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return -1; - } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; - } - return ts; -} - -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { - int64_t ts = 0; - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - // uError("SML:data:%s,len:%d", data, len); - ts = smlParseInfluxTime(info, data, len); - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ts = smlParseOpenTsdbTime(info, data, len); - } else { - ASSERT(0); - } - uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - // number - if (smlParseNumber(pVal, msg)) { - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - - return TSDB_CODE_TSC_INVALID_VALUE; -} - -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; - elements->measure = sql; - - // parse measure - while (sql < sqlEnd) { - if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; - continue; - } - if (IS_COMMA(sql)) { - break; - } - - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; - } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->tagsLen = sql - elements->tags; - } - elements->measureTagsLen = sql - elements->measure; - - // parse cols - JUMP_SPACE(sql, sqlEnd) - elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; - } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; - } - elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; - } - - // parse timestamp - JUMP_SPACE(sql, sqlEnd) - elements->timestamp = sql; - while (sql < sqlEnd) { - if (isspace(*sql)) { - break; - } - sql++; - } - elements->timestampLen = sql - elements->timestamp; - - return TSDB_CODE_SUCCESS; -} - -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { - while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { - *data = *sql; - } else if (**sql == SPACE && *data) { - *len = *sql - *data; - break; - } - (*sql)++; - } -} - -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); - while (sql < sqlEnd) { - JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; - - const char *key = sql; - int32_t keyLen = 0; - - // parse key - while (sql < sqlEnd) { - if (*sql == SPACE) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (*sql == EQUAL) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - while (sql < sqlEnd) { - // parse value - if (*sql == SPACE) { - break; - } - if (*sql == EQUAL) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; - - taosArrayPush(cols, &kv); - } - - return TSDB_CODE_SUCCESS; -} - -// format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - - // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; - } - - // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { - return ret; - } - - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { - return TSDB_CODE_SUCCESS; - } - - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; - - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; - } - if (!isInQuote && IS_COMMA(sql)) { - break; - } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - sql++; - - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) - - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); - - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - - int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); - if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; - } - } - } - } else { - size_t tmp = taosArrayGetSize(metaArray); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - } - } - - return TSDB_CODE_SUCCESS; -} - -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} - -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; - } - - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; - -cleanup: - taosMemoryFree(tag); - return NULL; -} - -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); - } - taosArrayDestroy(kvArray); - } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); - } - taosHashCleanup(kvHash); - } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } - taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); - taosMemoryFree(tag); -} - -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} - -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); +/* +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; } - return TSDB_CODE_SUCCESS; - } - - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - } - - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + end: + taosHashCleanup(dumplicateKey); + return ret; +} +*/ - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols){ + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + if(ret == 0){ + taosArrayPush(metaArray, kv); + } } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; } static void smlDestroySTableMeta(SSmlSTableMeta *meta) { taosHashCleanup(meta->tagHash); taosHashCleanup(meta->colHash); taosArrayDestroy(meta->tags); - taosArrayDestroy(meta->cols); - taosMemoryFree(meta->tableMeta); - taosMemoryFree(meta); -} - -static void smlDestroyCols(SArray *cols) { - if (!cols) return; - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - qDestroyQuery(info->pQuery); - - // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); - } - taosHashCleanup(info->childTables); - - // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); - } - taosHashCleanup(info->superTables); - - // destroy info->pVgHash - taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); - } - destroyRequest(info->pRequest); - - cJSON_Delete(info->root); - taosMemoryFreeClear(info); -} - -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return NULL; - } - info->id = smlGenId(); - - info->pQuery = smlInitHandle(); - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } - } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - } - - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->pQuery || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - - return info; -cleanup: - smlDestroyInfo(info); - return NULL; -} - -/************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - tinfo->sTableName = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - *tsVal = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'u': - case 'U': - // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'n': - case 'N': - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { - // Timestamp must be the first KV to parse - int64_t tsVal = 0; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - - return TSDB_CODE_SUCCESS; + taosArrayDestroy(meta->cols); + taosMemoryFree(meta->tableMeta); + taosMemoryFree(meta); } -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { + for (int i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if (index) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag){ + if (kv->length > value->length) { + value->length = kv->length; + } + continue; + } + if (kv->type != value->type) { + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return TSDB_CODE_SML_NOT_SAME_TYPE; + } + + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; + } + } else { + size_t tmp = taosArrayGetSize(metaArray); + ASSERT(tmp <= INT16_MAX); + int16_t size = tmp; + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); + if(ret == 0){ + taosArrayPush(metaArray, kv); } - break; } - default: - return TSDB_CODE_TSC_INVALID_JSON; } return TSDB_CODE_SUCCESS; } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; +static void smlDestroyTableInfo(SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - return TSDB_CODE_SUCCESS; + taosMemoryFree(tag->key); + taosArrayDestroy(tag->cols); + taosArrayDestroy(tag->tags); + taosMemoryFree(tag); } -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { - int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; - } - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; - } +void smlDestroyInfo(SSmlHandle *info) { + if (!info) return; + qDestroyQuery(info->pQuery); - size_t childTableNameLen = strlen(tsSmlChildTableName); - int32_t tagNum = cJSON_GetArraySize(tags); - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; + // destroy info->childTables + NodeList* tmp = info->childTables; + while (tmp) { + if(tmp->data.used) { + smlDestroyTableInfo((SSmlTableInfo*)tmp->data.value); } + NodeList* t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; - } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; + // destroy info->superTables + tmp = info->superTables; + while (tmp) { + if(tmp->data.used) { + smlDestroySTableMeta((SSmlSTableMeta*)tmp->data.value); } + NodeList* t = tmp->next; + taosMemoryFree(tmp); + tmp = t; + } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); - // key - kv->keyLen = keyLen; - kv->key = tag->string; + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->preLineColKV); - // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if(!info->dataFormat){ + for(int i = 0; i < info->lineNum; i++){ + taosArrayDestroy(info->lines[i].colArray); } + taosMemoryFree(info->lines); } - return ret; + taosMemoryFreeClear(info); } -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { - int32_t ret = TSDB_CODE_SUCCESS; - - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; +SSmlHandle *smlBuildSmlInfo(TAOS *taos) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; } - - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + if(taos != NULL){ + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; + } } - // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; + if (NULL == info->pVgHash) { + uError("create SSmlHandle failed"); + goto cleanup; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); - // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; + return info; + +cleanup: + smlDestroyInfo(info); + return NULL; +} + +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!kvHash) { + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_OUT_OF_MEMORY; + } + for (size_t i = 0; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -/************* TSDB_SML_JSON_PROTOCOL function end **************/ -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql)); +static int32_t smlParseLineBottom(SSmlHandle *info) { + if(info->dataFormat) return TSDB_CODE_SUCCESS; - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } - - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + for(int32_t i = 0; i < info->lineNum; i ++){ + SSmlLineInfo* elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if(info->protocol == TSDB_SML_LINE_PROTOCOL){ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements->measure, elements->measureTagsLen, NULL); + }else if(info->protocol == TSDB_SML_TELNET_PROTOCOL){ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + }else{ + tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + if(tinfo == NULL){ + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName, 0}; - - buildChildTableName(&rName); - (*oneTable)->uid = rName.uid; - } else { - (*oneTable)->uid = *(uint64_t *)((*oneTable)->childTableName); - } - } - - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } + int ret = smlPushCols(tinfo->cols, elements->colArray); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); return ret; } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName, 0}; - buildChildTableName(&rName); - tinfo->uid = rName.uid; - } else { - tinfo->uid = *(uint64_t *)(tinfo->childTableName); // generate uid by name simple - } - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + if (tableMeta) { // update meta + ret = smlUpdateMeta(tableMeta->colHash, tableMeta->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta(tableMeta->tagHash, tableMeta->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + } else { +// ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); +// if (ret != TSDB_CODE_SUCCESS) { +// uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); +// return ret; +// } - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + nodeListSet(&info->superTables, elements->measure, elements->measureLen, meta, NULL); } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (info->root == NULL) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; - } - } - -end: - return ret; -} static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; - SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); - while (oneTable) { - SSmlTableInfo *tableData = *oneTable; + NodeList* tmp = info->childTables; + while (tmp) { + SSmlTableInfo *tableData = (SSmlTableInfo *)tmp->data.value; SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); @@ -2313,22 +1201,22 @@ static int32_t smlInsertData(SSmlHandle *info) { } taosHashPut(info->pVgHash, (const char *)&vg.vgId, sizeof(vg.vgId), (char *)&vg, sizeof(vg)); - SSmlSTableMeta **pMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + SSmlSTableMeta *pMeta = + (SSmlSTableMeta *)nodeListGet(info->superTables, tableData->sTableName, tableData->sTableNameLen, NULL); + ASSERT(NULL != pMeta); // use tablemeta of stable to save vgid and uid of child table - (*pMeta)->tableMeta->vgId = vg.vgId; - (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid + pMeta->tableMeta->vgId = vg.vgId; + pMeta->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->pQuery, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, - (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, pMeta->cols, tableData->cols, + pMeta->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBindData failed", info->id); return code; } - oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + tmp = tmp->next; } code = smlBuildOutput(info->pQuery, info->pVgHash); @@ -2338,27 +1226,18 @@ static int32_t smlInsertData(SSmlHandle *info) { } info->cost.insertRpcTime = taosGetTimestampUs(); - // launchQueryImpl(info->pRequest, info->pQuery, false, NULL); - // info->affectedRows = taos_affected_rows(info->pRequest); - // return info->pRequest->code; - SAppClusterSummary *pActivity = &info->taos->pAppInfo->summary; atomic_add_fetch_64((int64_t *)&pActivity->numOfInsertsReq, 1); - SSqlCallbackWrapper *pWrapper = (SSqlCallbackWrapper *)taosMemoryCalloc(1, sizeof(SSqlCallbackWrapper)); - if (pWrapper == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pWrapper->pRequest = info->pRequest; - launchAsyncQuery(info->pRequest, info->pQuery, NULL, pWrapper); - return TSDB_CODE_SUCCESS; + launchQueryImpl(info->pRequest, info->pQuery, true, NULL); + return info->pRequest->code; } static void smlPrintStatisticInfo(SSmlHandle *info) { uError("SML:0x%" PRIx64 - " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ + " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", + "", info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, @@ -2366,6 +1245,44 @@ static void smlPrintStatisticInfo(SSmlHandle *info) { info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); } +int32_t smlClearForRerun(SSmlHandle *info){ + info->reRun = false; + // clear info->childTables + NodeList* pList = info->childTables; + while (pList) { + if(pList->data.used) { + smlDestroyTableInfo((SSmlTableInfo*)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + // clear info->superTables + pList = info->superTables; + while (pList) { + if(pList->data.used) { + smlDestroySTableMeta((SSmlSTableMeta*)pList->data.value); + pList->data.used = false; + } + pList = pList->next; + } + + if(unlikely(info->lines != NULL)){ + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = (SSmlLineInfo*)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifOpStmt* stmt= (SVnodeModifOpStmt*)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return TSDB_CODE_SUCCESS; +} + static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { int32_t code = TSDB_CODE_SUCCESS; if (info->protocol == TSDB_SML_JSON_PROTOCOL) { @@ -2381,7 +1298,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2400,10 +1319,23 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if(info->dataFormat){ + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + }else{ + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if(info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + }else{ + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } + } else { ASSERT(0); } @@ -2411,7 +1343,18 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if(info->reRun){ + i = 0; + rawLine = oldRaw; + code = smlClearForRerun(info); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + continue; + } + i++; } + return code; } @@ -2426,17 +1369,22 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } - info->cost.lineNum = numLines; - info->cost.numOfSTables = taosHashGetSize(info->superTables); - info->cost.numOfCTables = taosHashGetSize(info->childTables); + info->cost.lineNum = info->lineNum; + info->cost.numOfSTables = nodeListSize(info->superTables); + info->cost.numOfCTables = nodeListSize(info->childTables); info->cost.schemaTime = taosGetTimestampUs(); do { code = smlModifyDBSchemas(info); if (code == 0) break; - } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); + } while (retryNum++ < nodeListSize(info->superTables) * MAX_RETRY_TIMES); if (code != 0) { uError("SML:0x%" PRIx64 " smlModifyDBSchemas error : %s", info->id, tstrerror(code)); @@ -2453,92 +1401,42 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertCallback(void *param, void *res, int32_t code) { - SRequestObj *pRequest = (SRequestObj *)res; - SSmlHandle *info = (SSmlHandle *)param; - int32_t rows = taos_affected_rows(pRequest); - - uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); - Params *pParam = info->params; - // lock - taosThreadSpinLock(&pParam->lock); - pParam->cnt++; - if (code != TSDB_CODE_SUCCESS) { - pParam->request->code = code; - pParam->request->body.resInfo.numOfRows += rows; - } else { - pParam->request->body.resInfo.numOfRows += info->affectedRows; +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, + int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; } - // unlock - taosThreadSpinUnlock(&pParam->lock); - if (pParam->cnt == pParam->total) { - tsem_post(&pParam->sem); + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; } - uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); - info->cost.endTime = taosGetTimestampUs(); - info->cost.code = code; - smlPrintStatisticInfo(info); - smlDestroyInfo(info); -} -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - int batchs = 0; - STscObj *pTscObj = request->pTscObj; + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; - pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - - Params params = {0}; - params.request = request; - tsem_init(¶ms.sem, 0, 0); - taosThreadSpinInit(&(params.lock), 0); - if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2560,65 +1458,14 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - batchs = ceil(((double)numLines) / tsSmlBatchSize); - params.total = batchs; - for (int i = 0; i < batchs; ++i) { - SRequestObj *req = (SRequestObj *)createRequest(pTscObj->id, TSDB_SQL_INSERT, 0); - if (!req) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error request is null"); - goto end; - } - SSmlHandle *info = smlBuildSmlInfo(pTscObj, req, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - - int32_t perBatch = tsSmlBatchSize; - - if (numLines > perBatch) { - numLines -= perBatch; - } else { - perBatch = numLines; - numLines = 0; - } - - info->params = ¶ms; - info->affectedRows = perBatch; - info->pRequest->body.queryFp = smlInsertCallback; - info->pRequest->body.param = info; - int32_t code = smlProcess(info, lines, rawLine, rawLineEnd, perBatch); - if (lines) { - lines += perBatch; - } - if (rawLine) { - int num = 0; - while (rawLine < rawLineEnd) { - if (*(rawLine++) == '\n') { - num++; - } - if (num == perBatch) { - break; - } - } - } - if (code != TSDB_CODE_SUCCESS) { - info->pRequest->body.queryFp(info, req, code); - } - } - tsem_wait(¶ms.sem); + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; + info->cost.endTime = taosGetTimestampUs(); + info->cost.code = code; +// smlPrintStatisticInfo(info); end: - taosThreadSpinDestroy(¶ms.lock); - tsem_destroy(¶ms.sem); - // ((STscObj *)taos)->schemalessType = 0; - pTscObj->schemalessType = 1; - uDebug("resultend:%s", request->msgBuf); + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2643,25 +1490,7 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { @@ -2677,25 +1506,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - + int precision, int32_t ttl, int64_t reqid) { int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2708,7 +1519,7 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c new file mode 100644 index 0000000000000000000000000000000000000000..3de1397f72533c2ee35df832116e5e81e45686e2 --- /dev/null +++ b/source/client/src/clientSmlJson.c @@ -0,0 +1,501 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "clientSml.h" + +#define JUMP_JSON_SPACE(start) \ +while(*(start)){\ + if(unlikely(*(start) > 32))\ + break;\ + else\ + (start)++;\ + } + +SArray *smlJsonParseTags(char *start, char *end){ + SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); + while(start < end){ + SSmlKv kv = {0}; + kv.type = TSDB_DATA_TYPE_NCHAR; + bool isInQuote = false; + while(start < end){ + if(unlikely(!isInQuote && *start == '"')){ + start++; + kv.key = start; + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *start == '"')){ + kv.keyLen = start - kv.key; + start++; + break; + } + start++; + } + bool hasColon = false; + while(start < end){ + if(unlikely(!hasColon && *start == ':')){ + start++; + hasColon = true; + continue; + } + if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ + kv.value = start; + start++; + continue; + } + + if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ + kv.length = start - kv.value; + taosArrayPush(tags, &kv); + start++; + break; + } + start++; + } + } + return tags; +} + +static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); + int32_t tagNum = taosArrayGetSize(tags); + for (int32_t i = 0; i < tagNum; ++i) { + SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + taosArrayDestroy(tags); + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; + } + taosArrayDestroy(tags); + + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return ret; +} + +static char* smlJsonGetObj(char *payload){ + int leftBracketCnt = 0; + while(*payload) { + if (unlikely(*payload == '{')) { + leftBracketCnt++; + payload++; + continue; + } + if (unlikely(*payload == '}')) { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; + } + continue; + } + payload++; + } + return NULL; +} + +void smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + char *sTmp = *start; + if((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' + && (*start)[4] == 'r' && (*start)[5] == 'i' && (*start)[6] == 'c' && (*start)[7] == '"'){ + + (*start) += 8; + bool isInQuote = false; + while(*(*start)){ + if(unlikely(!isInQuote && *(*start) == '"')){ + (*start)++; + offset[index++] = *start - sTmp; + element->measure = (*start); + isInQuote = true; + continue; + } + if(unlikely(isInQuote && *(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' + && (*start)[4] == 'e' && (*start)[5] == 's' && (*start)[6] == 't' + && (*start)[7] == 'a' && (*start)[8] == 'm' && (*start)[9] == 'p' && (*start)[10] == '"'){ + + (*start) += 11; + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->timestamp = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' + && (*start)[4] == 'u' && (*start)[5] == 'e' && (*start)[6] == '"'){ + + (*start) += 7; + + bool hasColon = false; + while(*(*start)){ + if(unlikely(!hasColon && *(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->cols = (*start); + hasColon = true; + continue; + } + if(unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' + && (*start)[4] == 's' && (*start)[5] == '"'){ + (*start) += 6; + + while(*(*start)){ + if(unlikely(*(*start) == ':')){ + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + (*start)++; + } + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } +} + +void smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset){ + int index = 0; + while(*(*start)){ + if((*start)[0] != '"'){ + (*start)++; + continue; + } + + if(unlikely(index >= 4)) { + uError("index >= 4, %s", *start) + break; + } + if((*start)[1] == 'm'){ + (*start) += offset[index++]; + element->measure = *start; + while(*(*start)){ + if(unlikely(*(*start) == '"')){ + element->measureLen = (*start) - element->measure; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'i'){ + (*start) += offset[index++]; + element->timestamp = *start; + while(*(*start)){ + if(unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + }else if((*start)[1] == 'v'){ + (*start) += offset[index++]; + element->cols = *start; + while(*(*start)){ + if(unlikely( *(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)){ + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + }else if((*start)[1] == 't' && (*start)[2] == 'a'){ + (*start) += offset[index++]; + element->tags = (*start); + char* tmp = smlJsonGetObj((*start)); + if(tmp){ + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + if(*(*start) == '}'){ + (*start)++; + break; + } + (*start)++; + } +} + +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if(info->offset[0] == 0){ + smlJsonParseObjFirst(start, elements, info->offset); + }else{ + smlJsonParseObj(start, elements, info->offset); + } + if(**start == '\0') return TSDB_CODE_SUCCESS; + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseNumber(&kv, &info->msgBuf)) { + kv.length = (int16_t)tDataTypes[kv.type].bytes; + }else{ + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // Parse tags + ret = smlParseTagsFromJSON(info, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if(info->dataFormat){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSON(SSmlHandle *info, char *payload) { + int32_t payloadNum = 1 << 15; + int32_t ret = TSDB_CODE_SUCCESS; + + int cnt = 0; + char *dataPointStart = payload; + while (1) { + if(info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, &dataPointStart, &element); + }else{ + if(cnt >= payloadNum){ + payloadNum = payloadNum << 1; + void* tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if(tmp != NULL){ + info->lines = (SSmlLineInfo*)tmp; + } + } + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); + return ret; + } + + if(*dataPointStart == '\0') break; + + if(unlikely(info->reRun)){ + cnt = 0; + dataPointStart = payload; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if(ret != TSDB_CODE_SUCCESS){ + return ret; + } + continue; + } + cnt++; + } + info->lineNum = cnt; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c new file mode 100644 index 0000000000000000000000000000000000000000..f56ad34fc52bc1687ce42aa38256b01eb4724f68 --- /dev/null +++ b/source/client/src/clientSmlLine.c @@ -0,0 +1,700 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +// comma , +//#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) +// space +//#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) +// equal = +//#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +// quote " +//#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +// SLASH +//#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) + +#define IS_SLASH_LETTER(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || *(sql) == SLASH)) \ +// (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + i--; \ + keyLen--; \ + } \ + } + +#define BINARY_ADD_LEN 2 // "binary" 2 means " " +#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; + +static bool smlParseBool(SSmlKv *kvVal) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { + kvVal->i = TSDB_TRUE; + return true; + } + + if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { + kvVal->i = TSDB_FALSE; + return true; + } + + if ((len == 4) && !strncasecmp(pVal, "true", len)) { + kvVal->i = TSDB_TRUE; + return true; + } + if ((len == 5) && !strncasecmp(pVal, "false", len)) { + kvVal->i = TSDB_FALSE; + return true; + } + return false; +} + +static bool smlIsBinary(const char *pVal, uint16_t len) { + // binary: "abc" + if (len < 2) { + return false; + } + if (pVal[0] == '"' && pVal[len - 1] == '"') { + return true; + } + return false; +} + +static bool smlIsNchar(const char *pVal, uint16_t len) { + // nchar: L"abc" + if (len < 3) { + return false; + } + if (pVal[1] == '"' && pVal[len - 1] == '"' && (pVal[0] == 'l' || pVal[0] == 'L')) { + return true; + } + return false; +} + +static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if(unlikely(len == 0 || (len == 1 && data[0] == '0'))){ + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; + + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + if (pVal->value[0] == '"'){ // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if(pVal->value[0] == 'l' || pVal->value[0] == 'L'){ // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3){ + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T'){ + if(pVal->length == 1 || (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') + && (pVal->value[2] == 'u' || pVal->value[2] == 'U') + && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))){ + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F'){ + if(pVal->length == 1 || (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') + && (pVal->value[2] == 'l' || pVal->value[2] == 'L') + && (pVal->value[3] == 's' || pVal->value[3] == 'S') + && (pVal->value[4] == 'e' || pVal->value[4] == 'E'))){ + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // number + if (smlParseNumber(pVal, msg)) { + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_TSC_INVALID_VALUE; +} + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + if(isSameCTable){ + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + }else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + void* oneTable = nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if(info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if(tinfo->tableDataCtx == NULL){ + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + nodeListSet(&info->childTables, currElement->measure, currElement->measureTagsLen, tinfo, NULL); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, + SSmlLineInfo* currElement, bool isSameMeasure, bool isSameCTable){ + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(unlikely(!isSameCTable)){ + SSmlTableInfo *oneTable = (SSmlTableInfo *)nodeListGet(info->childTables, currElement->measure, currElement->measureTagsLen, NULL); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = oneTable->tableDataCtx; + } + + if(unlikely(!isSameMeasure)){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, currElement->measure, currElement->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->cols; + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_QUOTE(*sql))) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote){ + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if(!hasSlash){ + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if(unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + + if(info->dataFormat){ + //cnt begin 0, add ts so + 2 + if(unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(kv.type != preKV->type){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, currElement->measure, currElement->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.type != preKV->type)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(IS_VAR_DATA_TYPE(kv.type)){ + if(kv.length > preKV->length) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + if(currElement->colArray == NULL){ + currElement->colArray = taosArrayInit(16, sizeof(SSmlKv)); + taosArraySetSize(currElement->colArray, 1); + } + taosArrayPush(currElement->colArray, &kv); //reserve for timestamp + } + + cnt++; + if(IS_SPACE(*sql)){ + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; + elements->measure = sql; + + // parse measure + while (sql < sqlEnd) { + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; + continue; + } + if (unlikely(IS_COMMA(sql))) { + break; + } + + if (unlikely(IS_SPACE(sql))) { + break; + } + sql++; + } + elements->measureLen = sql - elements->measure; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // to get measureTagsLen before + const char* tmp = sql; + while (tmp < sqlEnd){ + if (unlikely(IS_SPACE(tmp))) { + break; + } + tmp++; + } + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if(IS_SAME_CHILD_TABLE){ + isSameCTable = true; + isSameMeasure = true; + }else if(info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + elements->tagsLen = sql - elements->tags; + + // parse cols + JUMP_SPACE(sql, sqlEnd) + elements->cols = sql; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if(unlikely(ret != TSDB_CODE_SUCCESS)){ + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + elements->colsLen = sql - elements->cols; + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse timestamp + JUMP_SPACE(sql, sqlEnd) + elements->timestamp = sql; + while (sql < sqlEnd) { + if (unlikely(isspace(*sql))) { + break; + } + sql++; + } + elements->timestampLen = sql - elements->timestamp; + + int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts <= 0)) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if(info->dataFormat){ + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + }else{ + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; +} + diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c new file mode 100644 index 0000000000000000000000000000000000000000..53a7e3e81e99b8d3a5107964fe7a15c41a2a2420 --- /dev/null +++ b/source/client/src/clientSmlTelnet.c @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +int32_t is_same_child_table_telnet(const void *a, const void *b){ + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) + && ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) ? 0 : 1; +} + +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs()/smlFactorNS[toPrecision]; + } + int8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + + +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { + while (*sql < sqlEnd) { + if (unlikely((**sql != SPACE && !(*data)))) { + *data = *sql; + } else if (unlikely(**sql == SPACE && *data)) { + *len = *sql - *data; + break; + } + (*sql)++; + } +} + +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if(is_same_child_table_telnet(elements, &info->preLine) == 0){ + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if(info->dataFormat){ + if(!isSameMeasure){ + SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + + if(unlikely(sMeta == NULL)){ + sMeta = smlBuildSTableMeta(info->dataFormat); + STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + sMeta->tableMeta = pTableMeta; + if(pTableMeta == NULL){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); + } + info->currSTableMeta = sMeta->tableMeta; + superKV = sMeta->tags; + + if(unlikely(taosArrayGetSize(superKV) == 0)){ + isSuperKVInit = false; + } + taosArraySetSize(preLineKV, 0); + } + }else{ + taosArraySetSize(preLineKV, 0); + } + + const char *sql = data; + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == '\0')) break; + + const char *key = sql; + size_t keyLen = 0; + + // parse key + while (sql < sqlEnd) { + if (unlikely(*sql == SPACE)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(*sql == EQUAL)) { + keyLen = sql - key; + sql++; + break; + } + sql++; + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } +// if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { +// smlBuildInvalidDataMsg(msg, "dumplicate key", key); +// return TSDB_CODE_TSC_DUP_NAMES; +// } + + // parse value + const char *value = sql; + size_t valueLen = 0; + while (sql < sqlEnd) { + // parse value + if (unlikely(*sql == SPACE)) { + break; + } + if (unlikely(*sql == EQUAL)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + sql++; + } + valueLen = sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + + if(info->dataFormat){ + if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if(isSameMeasure){ + if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if(unlikely(kv.length > preKV->length)){ + preKV->length = kv.length; + SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, elements->measureLen, NULL); + ASSERT(tableMeta != NULL); + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + if(isSuperKVInit){ + if(unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if(unlikely(kv.length > preKV->length)) { + preKV->length = kv.length; + }else{ + kv.length = preKV->length; + } + info->needModifySchema = true; + + if(unlikely(!IS_SAME_KEY)){ + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + }else{ + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + }else{ + taosArrayPush(preLineKV, &kv); + } + cnt++; + } + SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, is_same_child_table_telnet); + if (unlikely(tinfo == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + *key = *elements; + tinfo->key = key; + nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); + } + if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; + + return TSDB_CODE_SUCCESS; +} + +// format: =[ =] +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + + // parse metric + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // parse timestamp + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if(info->dataFormat && info->currSTableMeta == NULL){ + needConverTime = true; + } + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = { .key = TS, .keyLen = TS_LEN, .type = TSDB_DATA_TYPE_TIMESTAMP, .i = ts, .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + // parse value + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + if (unlikely(!elements->tags || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if(unlikely(info->reRun)){ + return TSDB_CODE_SUCCESS; + } + + if(info->dataFormat){ + if(needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); + } + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if(ret == TSDB_CODE_SUCCESS){ + ret = smlBuildRow(info->currTableDataCtx); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + }else{ + if(elements->colArray == NULL){ + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index b9887f28d4896d1724e9d6529fd8297c83c1f9be..7bc99c65e8662685a32a24779389619f0c2ae422 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -83,7 +83,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { } break; default: - code = TSDB_CODE_TSC_APP_ERROR; + code = TSDB_CODE_APP_ERROR; break; } @@ -118,7 +118,7 @@ int32_t stmtBackupQueryFields(STscStmt* pStmt) { pRes->fields = taosMemoryMalloc(size); pRes->userFields = taosMemoryMalloc(size); if (NULL == pRes->fields || NULL == pRes->userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pRes->fields, pStmt->exec.pRequest->body.resInfo.fields, size); memcpy(pRes->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size); @@ -136,7 +136,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { pStmt->exec.pRequest->body.resInfo.fields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.fields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.fields, pRes->fields, size); } @@ -144,7 +144,7 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { pStmt->exec.pRequest->body.resInfo.userFields = taosMemoryMalloc(size); if (NULL == pStmt->exec.pRequest->body.resInfo.userFields) { - STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY); + STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pStmt->exec.pRequest->body.resInfo.userFields, pRes->userFields, size); } @@ -152,9 +152,10 @@ int32_t stmtRestoreQueryFields(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, bool autoCreateTbl) { +int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SName* tbName, const char* sTableName, + bool autoCreateTbl) { STscStmt* pStmt = (STscStmt*)stmt; - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(tbName, tbFName); memcpy(&pStmt->bInfo.sname, tbName, sizeof(*tbName)); @@ -171,12 +172,11 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; - pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -186,7 +186,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SNam STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -241,6 +241,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } int32_t stmtParseSql(STscStmt* pStmt) { + pStmt->exec.pCurrBlock = NULL; + SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, @@ -293,14 +295,14 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); -/* - if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { + if (keepTable && pBlocks == pStmt->exec.pCurrBlock) { + ASSERT(NULL == pBlocks->pData); + TSWAP(pBlocks->pData, pStmt->exec.pCurrTbData); STMT_ERR_RET(qResetStmtDataBlock(pBlocks, false)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; } -*/ qDestroyStmtDataBlock(pBlocks); taosHashRemove(pStmt->exec.pBlockHash, key, keyLen); @@ -308,13 +310,16 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } - pStmt->exec.autoCreateTbl = false; - - if (!keepTable) { - taosHashCleanup(pStmt->exec.pBlockHash); - pStmt->exec.pBlockHash = NULL; + if (keepTable) { + return TSDB_CODE_SUCCESS; } + taosHashCleanup(pStmt->exec.pBlockHash); + pStmt->exec.pBlockHash = NULL; + + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -350,7 +355,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid) { +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, + uint64_t suid) { SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -362,7 +368,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableD STMT_ERR_RET( taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgInfo.vgId, pStmt->sql.autoCreateTbl)); + + STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgInfo.vgId); return TSDB_CODE_SUCCESS; } @@ -371,12 +379,13 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - STableDataCxt* pCxtInExec = - taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pCxtInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (pCxtInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; + pStmt->exec.pCurrBlock = *pCxtInExec; + if (pStmt->sql.autoCreateTbl) { tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -403,18 +412,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - pStmt->exec.autoCreateTbl = true; - pStmt->bInfo.tbUid = 0; STableDataCxt* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0)); + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0, pStmt->bInfo.tbSuid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64, pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); return TSDB_CODE_SUCCESS; @@ -459,7 +468,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { tscError("table [%s, %" PRIx64 ", %" PRIx64 "] found in exec blockHash, but not in sql blockHash", pStmt->bInfo.tbFName, uid, cacheUid); - STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.needParse = false; @@ -486,13 +495,15 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.tagsCached = true; STableDataCxt* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid)); + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid, suid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -508,7 +519,7 @@ int32_t stmtResetStmt(STscStmt* pStmt) { pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; STMT_ERR_RET(terrno); } @@ -523,13 +534,13 @@ TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid) { pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (NULL == pStmt) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pStmt->sql.pTableCache = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == pStmt->sql.pTableCache) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pStmt); return NULL; } @@ -614,7 +625,7 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } tscDebug("start to bind stmt tag values"); @@ -622,8 +633,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->exec.autoCreateTbl = true; - return TSDB_CODE_SUCCESS; } @@ -637,7 +646,7 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtTagFields(*pDataBlock, pStmt->bInfo.boundTags, fieldNum, fields)); @@ -655,7 +664,7 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } STMT_ERR_RET(qBuildStmtColFields(*pDataBlock, fieldNum, fields)); @@ -725,11 +734,18 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { return TSDB_CODE_SUCCESS; } - STableDataCxt** pDataBlock = - (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STableDataCxt** pDataBlock = NULL; + + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + pStmt->exec.pCurrBlock = *pDataBlock; } if (colIdx < 0) { @@ -741,7 +757,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } else { if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); - STMT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + STMT_ERR_RET(TSDB_CODE_APP_ERROR); } pStmt->bInfo.sBindLastIdx = colIdx; @@ -775,10 +791,10 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { int32_t code = 0; int32_t finalCode = 0; size_t keyLen = 0; - void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { STableDataCxt* pBlock = *(STableDataCxt**)pIter; - char* key = taosHashGetKey(pIter, &keyLen); + char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid) { @@ -844,7 +860,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { pMeta->uid = pTableMeta->uid; pStmt->bInfo.tbUid = pTableMeta->uid; - taosMemoryFree(pTableMeta); + taosMemoryFree(pTableMeta); } pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); @@ -857,7 +873,6 @@ int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; SSubmitRsp* pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_DLOG_E("start to exec"); @@ -866,8 +881,13 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + + STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -890,15 +910,6 @@ _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { - if (NULL == pRsp) { - tscError("no submit resp got for auto create table"); - code = TSDB_CODE_TSC_APP_ERROR; - } else { - code = stmtUpdateTableUid(pStmt, pRsp); - } - } - tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index ade0c952276f0091a243d43d92a99c02a4f6cb57..350ecdd3736ea9415bce7e1a45f23c1ecd953293 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -691,7 +691,7 @@ void tmqAssignAskEpTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); *pTaskType = TMQ_DELAYED_TASK__ASK_EP; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -703,7 +703,7 @@ void tmqAssignDelayedCommitTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); *pTaskType = TMQ_DELAYED_TASK__COMMIT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -715,7 +715,7 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { - int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); + int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); *pTaskType = TMQ_DELAYED_TASK__REPORT; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); @@ -814,24 +814,55 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { return 0; } +static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { + // do nothing + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { + SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; + tDeleteSMqAskEpRsp(&pEpRspWrapper->msg); + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RSP) { + SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree); + taosArrayDestroy(pRsp->dataRsp.blockDataLen); + taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree); + taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosMemoryFree(pRsp->metaRsp.metaRsp); + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { + SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; + taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree); + taosArrayDestroy(pRsp->taosxRsp.blockDataLen); + taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree); + taosArrayDestroyP(pRsp->taosxRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + // taosx + taosArrayDestroy(pRsp->taosxRsp.createTableLen); + taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree); + } +} + void tmqClearUnhandleMsg(tmq_t* tmq) { - SMqRspWrapper* msg = NULL; + SMqRspWrapper* rspWrapper = NULL; while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else + taosGetQitem(tmq->qall, (void**)&rspWrapper); + if (rspWrapper) { + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); + } else { break; + } } - msg = NULL; + rspWrapper = NULL; taosReadAllQitems(tmq->mqueue, tmq->qall); while (1) { - taosGetQitem(tmq->qall, (void**)&msg); - if (msg) - taosFreeQitem(msg); - else + taosGetQitem(tmq->qall, (void**)&rspWrapper); + if (rspWrapper) { + tmqFreeRspWrapper(rspWrapper); + taosFreeQitem(rspWrapper); + } else { break; + } } } @@ -875,6 +906,7 @@ void tmqFreeImpl(void* handle) { tmq_t* tmq = (tmq_t*)handle; // TODO stop timer + tmqClearUnhandleMsg(tmq); if (tmq->mqueue) taosCloseQueue(tmq->mqueue); if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask); if (tmq->qall) taosFreeQall(tmq->qall); @@ -884,8 +916,7 @@ void tmqFreeImpl(void* handle) { int32_t sz = taosArrayGetSize(tmq->clientTopics); for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); + taosMemoryFreeClear(pTopic->schema.pSchema); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1004,7 +1035,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = -1; - tscDebug("call tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); + tscDebug("tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); @@ -1012,7 +1043,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; - tscDebug("call tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); + tscDebug("tmq subscribe, consumer: %" PRId64 ", topic num %d", tmq->consumerId, sz); for (int32_t i = 0; i < sz; i++) { char* topic = taosArrayGetP(container, i); @@ -1140,7 +1171,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { goto CREATE_MSG_FAIL; } if (code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0); if (pRspWrapper == NULL) { tscWarn("msg discard from vgId:%d, epoch %d since out of memory", vgId, epoch); goto CREATE_MSG_FAIL; @@ -1173,7 +1204,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { // handle meta rsp int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM, 0); if (pRspWrapper == NULL) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1215,6 +1246,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); + tscDebug("consumer:%" PRId64 ", put poll res into mqueue %p", tmq->consumerId, pRspWrapper); + taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1304,7 +1337,6 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { for (int32_t i = 0; i < sz; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (pTopic->schema.nCols) taosMemoryFreeClear(pTopic->schema.pSchema); - int32_t vgSz = taosArrayGetSize(pTopic->vgs); taosArrayDestroy(pTopic->vgs); } taosArrayDestroy(tmq->clientTopics); @@ -1362,7 +1394,7 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { tmqUpdateEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } else { - SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM); + SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); if (pWrapper == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; @@ -1410,7 +1442,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return -1; } void* pReq = taosMemoryCalloc(1, tlen); - if (tlen < 0) { + if (pReq == NULL) { tscError("failed to malloc askEpReq msg, size:%d", tlen); return -1; } @@ -1538,7 +1570,6 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper) { } int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { - /*tscDebug("call poll");*/ for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { @@ -1643,6 +1674,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) tDeleteSMqAskEpRsp(rspMsg); *pReset = true; } else { + tmqFreeRspWrapper(rspWrapper); *pReset = false; } } else { @@ -1665,6 +1697,8 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } } + tscDebug("consumer:%" PRId64 " handle rsp %p", tmq->consumerId, rspWrapper); + if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__END_RSP) { taosFreeQitem(rspWrapper); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; @@ -1692,6 +1726,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } else { tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", pollRspWrapper->dataRsp.head.epoch, consumerEpoch); + tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); } } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { @@ -1710,6 +1745,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } else { tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", pollRspWrapper->metaRsp.head.epoch, consumerEpoch); + tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); } } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__TAOSX_RSP) { @@ -1738,8 +1774,9 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); + tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); } } else { @@ -1756,7 +1793,6 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { - /*tscDebug("call poll1");*/ void* rspObj; int64_t startTime = taosGetTimestampMs(); @@ -1791,7 +1827,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { while (1) { tmqHandleAllDelayedTask(tmq); if (tmqPollImpl(tmq, timeout) < 0) { - tscDebug("return since poll err"); + tscDebug("consumer:%" PRId64 " return since poll err", tmq->consumerId); /*return NULL;*/ } diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 42988b16fe5e9dce30a5e295288a65794e940098..ccfc4980bc13d2c89c5d1be4554098257e7b1d7c 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -303,7 +303,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon); + jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 54778e87a788190c55fafc0e4d0db732677d795d..d608447506778cad1e964cdd79183d87be0a88e8 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../src/clientSml.c" +#include "../inc/clientSml.h" #include "taos.h" int main(int argc, char **argv) { @@ -40,11 +40,14 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -58,28 +61,23 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - - // case 3 false - tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); - ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); - ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -93,12 +91,14 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -111,91 +111,104 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + smlClearForRerun(info); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + taosMemoryFree(sql); + smlDestroyInfo(info); + } TEST(testCase, smlParseCols_Error_Test) { - const char *data[] = {"c=\"89sd", // binary, nchar - "c=j\"89sd\"", - "c=\"89sd\"k", - "c=u", // bool - "c=truet", - "c=f64", // double - "c=8f64f", - "c=8ef64", - "c=f32", // float - "c=8f32f", - "c=8wef32", - "c=-3.402823466e+39f32", - "c=", // double - "c=8f", - "c=8we", - "c=i8", // tiny int - "c=-8i8f", - "c=8wei8", - "c=-999i8", - "c=u8", // u tiny int - "c=8fu8", - "c=8weu8", - "c=999u8", - "c=-8u8", - "c=i16", // small int - "c=8fi16u", - "c=8wei16", - "c=-67787i16", - "c=u16", // u small int - "c=8u16f", - "c=8weu16", - "c=-9u16", - "c=67787u16", - "c=i32", // int - "c=8i32f", - "c=8wei32", - "c=2147483649i32", - "c=u32", // u int - "c=8u32f", - "c=8weu32", - "c=-4u32", - "c=42949672958u32", - "c=i64", // big int - "c=8i64i", - "c=8wei64", - "c=-9223372036854775809i64", - "c=i", // big int - "c=8fi", - "c=8wei", - "c=9223372036854775808i", - "c=u64", // u big int - "c=8u64f", - "c=8weu64", - "c=-3.402823466e+39u64", - "c=-339u64", - "c=18446744073709551616u64", - "c=1,c=2", - "c=1=2"}; - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + const char *data[] = {"st,t=1 c=\"89sd 1626006833639000000", // binary, nchar + "st,t=1 c=j\"89sd\" 1626006833639000000", + "st,t=1 c=\"89sd\"k 1626006833639000000", + "st,t=1 c=u 1626006833639000000", // bool + "st,t=1 c=truet 1626006833639000000", + "st,t=1 c=f64 1626006833639000000", // double + "st,t=1 c=8f64f 1626006833639000000", + "st,t=1 c=8ef64 1626006833639000000", + "st,t=1 c=f32 1626006833639000000", // float + "st,t=1 c=8f32f 1626006833639000000", + "st,t=1 c=8wef32 1626006833639000000", + "st,t=1 c=-3.402823466e+39f32 1626006833639000000", + "st,t=1 c= 1626006833639000000", // double + "st,t=1 c=8f 1626006833639000000", + "st,t=1 c=8we 1626006833639000000", + "st,t=1 c=i8 1626006833639000000", // tiny int + "st,t=1 c=-8i8f 1626006833639000000", + "st,t=1 c=8wei8 1626006833639000000", + "st,t=1 c=-999i8 1626006833639000000", + "st,t=1 c=u8 1626006833639000000", // u tiny int + "st,t=1 c=8fu8 1626006833639000000", + "st,t=1 c=8weu8 1626006833639000000", + "st,t=1 c=999u8 1626006833639000000", + "st,t=1 c=-8u8 1626006833639000000", + "st,t=1 c=i16 1626006833639000000", // small int + "st,t=1 c=8fi16u 1626006833639000000", + "st,t=1 c=8wei16 1626006833639000000", + "st,t=1 c=-67787i16 1626006833639000000", + "st,t=1 c=u16 1626006833639000000", // u small int + "st,t=1 c=8u16f 1626006833639000000", + "st,t=1 c=8weu16 1626006833639000000", + "st,t=1 c=-9u16 1626006833639000000", + "st,t=1 c=67787u16 1626006833639000000", + "st,t=1 c=i32 1626006833639000000", // int + "st,t=1 c=8i32f 1626006833639000000", + "st,t=1 c=8wei32 1626006833639000000", + "st,t=1 c=2147483649i32 1626006833639000000", + "st,t=1 c=u32 1626006833639000000", // u int + "st,t=1 c=8u32f 1626006833639000000", + "st,t=1 c=8weu32 1626006833639000000", + "st,t=1 c=-4u32 1626006833639000000", + "st,t=1 c=42949672958u32 1626006833639000000", + "st,t=1 c=i64 1626006833639000000", // big int + "st,t=1 c=8i64i 1626006833639000000", + "st,t=1 c=8wei64 1626006833639000000", + "st,t=1 c=-9223372036854775809i64 1626006833639000000", + "st,t=1 c=i 1626006833639000000", // big int + "st,t=1 c=8fi 1626006833639000000", + "st,t=1 c=8wei 1626006833639000000", + "st,t=1 c=9223372036854775808i 1626006833639000000", + "st,t=1 c=u64 1626006833639000000", // u big int + "st,t=1 c=8u64f 1626006833639000000", + "st,t=1 c=8weu64 1626006833639000000", + "st,t=1 c=-3.402823466e+39u64 1626006833639000000", + "st,t=1 c=-339u64 1626006833639000000", + "st,t=1 c=18446744073709551616u64 1626006833639000000", + "st,t=1 c=1=2 1626006833639000000"}; + + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -204,76 +217,14 @@ TEST(testCase, smlParseCols_Error_Test) { int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); - SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); - printf("i:%d\n", i); + SSmlLineInfo elements = {0}; + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); - taosHashClear(dumplicateKey); taosMemoryFree(sql); - for (int j = 0; j < taosArrayGetSize(cols); j++) { - void *kv = taosArrayGetP(cols, j); - taosMemoryFree(kv); - } - taosArrayDestroy(cols); + taosArrayDestroy(elements.colArray); } - taosHashCleanup(dumplicateKey); -} - -TEST(testCase, smlParseCols_tag_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - const char *data = - "cbin=\"passit " - "helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=" - "898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_" - "=l\"iuwq\""; - int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); - - // nchar - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 15); - ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 3); - ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 7); - ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); - - for (int i = 0; i < size; i++) { - void *tmp = taosArrayGetP(cols, i); - taosMemoryFree(tmp); - } - taosArrayClear(cols); - - // test tag is null - data = "t=3e"; - len = 0; - memset(msgBuf.buf, 0, msgBuf.len); - taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - size = taosArrayGetSize(cols); - ASSERT_EQ(size, 0); - - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + smlDestroyInfo(info); } TEST(testCase, smlParseCols_Test) { @@ -281,226 +232,207 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; + SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = - "cb\\=in=\"pass\\,it " + "st,t=1 cb\\=in=\"pass\\,it " "hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=" "233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t," - "cboolf=f,cnch_=l\"iuwq\""; + "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); + int32_t size = taosArrayGetSize(elements.colArray); + ASSERT_EQ(size, 20); // binary - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 8); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // double - kv = (SSmlKv *)taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->d, 4.31); printf("4.31 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->f, 8.32); printf("8.32 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); // ASSERT_EQ(kv->f, 8.23); printf("8.23 = kv->f:%f\n", kv->f); - taosMemoryFree(kv); // tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, -34); - taosMemoryFree(kv); // unsigned tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->u, 89); - taosMemoryFree(kv); // small int - kv = (SSmlKv *)taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 233); - taosMemoryFree(kv); // unsigned smallint - kv = (SSmlKv *)taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 898); - taosMemoryFree(kv); // int - kv = (SSmlKv *)taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 98289); - taosMemoryFree(kv); // unsigned int - kv = (SSmlKv *)taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 12323); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, 989); - taosMemoryFree(kv); // unsigned bigint - kv = (SSmlKv *)taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->u, 8989323); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosMemoryFree(kv); - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + taosArrayDestroy(elements.colArray); taosMemoryFree(sql); + smlDestroyInfo(info); } -TEST(testCase, smlGetTimestampLen_Test) { - uint8_t len = smlGetTimestampLen(0); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(390); - ASSERT_EQ(len, 3); - - len = smlGetTimestampLen(-1); - ASSERT_EQ(len, 1); - - len = smlGetTimestampLen(-10); - ASSERT_EQ(len, 2); - - len = smlGetTimestampLen(-390); - ASSERT_EQ(len, 3); -} +//TEST(testCase, smlGetTimestampLen_Test) { +// uint8_t len = smlGetTimestampLen(0); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(390); +// ASSERT_EQ(len, 3); +// +// len = smlGetTimestampLen(-1); +// ASSERT_EQ(len, 1); +// +// len = smlGetTimestampLen(-10); +// ASSERT_EQ(len, 2); +// +// len = smlGetTimestampLen(-390); +// ASSERT_EQ(len, 3); +//} TEST(testCase, smlParseNumber_Test) { SSmlKv kv = {0}; @@ -515,7 +447,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { @@ -532,479 +466,95 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 ", "sys.procs.running 1479496100 42 host= ", "sys.procs.running 1479496100 42or host=web01", - "sys.procs.running 1479496100 true host=web01", - "sys.procs.running 1479496100 \"binary\" host=web01", - "sys.procs.running 1479496100 L\"rfr\" host=web01", +// "sys.procs.running 1479496100 true host=web01", +// "sys.procs.running 1479496100 \"binary\" host=web01", +// "sys.procs.running 1479496100 L\"rfr\" host=web01", "sys.procs.running 1479496100 42 host=web01 cpu= ", - "sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01=er", "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_type_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = {"sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01", - "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 " - "id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 " - "t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\""}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); +TEST(testCase, smlParseTelnetLine_Test) { + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", + "twudyr 1626006833641 \"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" id=twudyr_17102_17825 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" t8=L\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\"", }; - - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - ASSERT_NE(ret, 0); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); + ASSERT_EQ(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_JSON_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", }; - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; + char *dataPointStart = (char *)sql[i]; + int8_t offset[4] = {0}; + while (1) { + SSmlLineInfo elements = {0}; + if(offset[0] == 0){ + smlJsonParseObjFirst(&dataPointStart, &elements, offset); + }else{ + smlJsonParseObj(&dataPointStart, &elements, offset); + } + if(*dataPointStart == '\0') break; + + SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); + size_t num = taosArrayGetSize(tags); + ASSERT_EQ(num, 1); + + taosArrayDestroy(tags); + } } - ASSERT_NE(ret, 0); smlDestroyInfo(info); } -TEST(testCase, sml_col_4096_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "spgwgvldxv,id=spgwgvldxv_1,t0=f " - "c0=t,c1=t,c2=t,c3=t,c4=t,c5=t,c6=t,c7=t,c8=t,c9=t,c10=t,c11=t,c12=t,c13=t,c14=t,c15=t,c16=t,c17=t,c18=t,c19=t," - "c20=t,c21=t,c22=t,c23=t,c24=t,c25=t,c26=t,c27=t,c28=t,c29=t,c30=t,c31=t,c32=t,c33=t,c34=t,c35=t,c36=t,c37=t,c38=" - "t,c39=t,c40=t,c41=t,c42=t,c43=t,c44=t,c45=t,c46=t,c47=t,c48=t,c49=t,c50=t,c51=t,c52=t,c53=t,c54=t,c55=t,c56=t," - "c57=t,c58=t,c59=t,c60=t,c61=t,c62=t,c63=t,c64=t,c65=t,c66=t,c67=t,c68=t,c69=t,c70=t,c71=t,c72=t,c73=t,c74=t,c75=" - "t,c76=t,c77=t,c78=t,c79=t,c80=t,c81=t,c82=t,c83=t,c84=t,c85=t,c86=t,c87=t,c88=t,c89=t,c90=t,c91=t,c92=t,c93=t," - "c94=t,c95=t,c96=t,c97=t,c98=t,c99=t,c100=t," - "c101=t,c102=t,c103=t,c104=t,c105=t,c106=t,c107=t,c108=t,c109=t,c110=t,c111=t,c112=t,c113=t,c114=t,c115=t,c116=t," - "c117=t,c118=t,c119=t,c120=t,c121=t,c122=t,c123=t,c124=t,c125=t,c126=t,c127=t,c128=t,c129=t,c130=t,c131=t,c132=t," - "c133=t,c134=t,c135=t,c136=t,c137=t,c138=t,c139=t,c140=t,c141=t,c142=t,c143=t,c144=t,c145=t,c146=t,c147=t,c148=t," - "c149=t,c150=t,c151=t,c152=t,c153=t,c154=t,c155=t,c156=t,c157=t,c158=t,c159=t,c160=t,c161=t,c162=t,c163=t,c164=t," - "c165=t,c166=t,c167=t,c168=t,c169=t,c170=t,c171=t,c172=t,c173=t,c174=t,c175=t,c176=t,c177=t,c178=t,c179=t,c180=t," - "c181=t,c182=t,c183=t,c184=t,c185=t,c186=t,c187=t,c188=t,c189=t," - "c190=t,c191=t,c192=t,c193=t,c194=t,c195=t,c196=t,c197=t,c198=t,c199=t,c200=t,c201=t,c202=t,c203=t,c204=t,c205=t," - "c206=t,c207=t,c208=t,c209=t,c210=t,c211=t,c212=t,c213=t,c214=t,c215=t,c216=t,c217=t,c218=t,c219=t,c220=t,c221=t," - "c222=t,c223=t,c224=t,c225=t,c226=t,c227=t,c228=t,c229=t,c230=t,c231=t,c232=t,c233=t,c234=t,c235=t,c236=t,c237=t," - "c238=t,c239=t,c240=t,c241=t,c242=t,c243=t,c244=t,c245=t,c246=t,c247=t,c248=t,c249=t,c250=t,c251=t,c252=t,c253=t," - "c254=t,c255=t,c256=t,c257=t,c258=t,c259=t,c260=t,c261=t,c262=t,c263=t,c264=t,c265=t,c266=t,c267=t,c268=t,c269=t," - "c270=t,c271=t,c272=t,c273=t,c274=t,c275=t,c276=t,c277=t,c278=t," - "c279=t,c280=t,c281=t,c282=t,c283=t,c284=t,c285=t,c286=t,c287=t,c288=t,c289=t,c290=t,c291=t,c292=t,c293=t,c294=t," - "c295=t,c296=t,c297=t,c298=t,c299=t,c300=t,c301=t,c302=t,c303=t,c304=t,c305=t,c306=t,c307=t,c308=t,c309=t,c310=t," - "c311=t,c312=t,c313=t,c314=t,c315=t,c316=t,c317=t,c318=t,c319=t,c320=t,c321=t,c322=t,c323=t,c324=t,c325=t,c326=t," - "c327=t,c328=t,c329=t,c330=t,c331=t,c332=t,c333=t,c334=t,c335=t,c336=t,c337=t,c338=t,c339=t,c340=t,c341=t,c342=t," - "c343=t,c344=t,c345=t,c346=t,c347=t,c348=t,c349=t,c350=t,c351=t,c352=t,c353=t,c354=t,c355=t,c356=t,c357=t,c358=t," - "c359=t,c360=t,c361=t,c362=t,c363=t,c364=t,c365=t,c366=t,c367=t,c368=t,c369=t,c370=t,c371=t,c372=t,c373=t,c374=t," - "c375=t,c376=t,c377=t,c378=t,c379=t,c380=t,c381=t,c382=t,c383=t,c384=t,c385=t,c386=t,c387=t,c388=t,c389=t,c390=t," - "c391=t,c392=t,c393=t,c394=t,c395=t,c396=t,c397=t,c398=t,c399=t,c400=t,c401=t,c402=t,c403=t,c404=t,c405=t,c406=t," - "c407=t,c408=t,c409=t,c410=t,c411=t,c412=t,c413=t,c414=t,c415=t,c416=t,c417=t,c418=t,c419=t,c420=t,c421=t,c422=t," - "c423=t,c424=t,c425=t,c426=t,c427=t,c428=t,c429=t,c430=t,c431=t,c432=t,c433=t,c434=t,c435=t,c436=t,c437=t,c438=t," - "c439=t,c440=t,c441=t,c442=t,c443=t,c444=t,c445=t,c446=t," - "c447=t,c448=t,c449=t,c450=t,c451=t,c452=t,c453=t,c454=t,c455=t,c456=t,c457=t,c458=t,c459=t,c460=t,c461=t,c462=t," - "c463=t,c464=t,c465=t,c466=t,c467=t,c468=t,c469=t,c470=t,c471=t,c472=t,c473=t,c474=t,c475=t,c476=t,c477=t,c478=t," - "c479=t,c480=t,c481=t,c482=t,c483=t,c484=t,c485=t,c486=t,c487=t,c488=t,c489=t,c490=t,c491=t,c492=t,c493=t,c494=t," - "c495=t,c496=t,c497=t,c498=t,c499=t,c500=t,c501=t,c502=t,c503=t,c504=t,c505=t,c506=t,c507=t,c508=t,c509=t,c510=t," - "c511=t,c512=t,c513=t,c514=t,c515=t,c516=t,c517=t,c518=t,c519=t,c520=t,c521=t,c522=t,c523=t,c524=t,c525=t,c526=t," - "c527=t,c528=t,c529=t,c530=t,c531=t,c532=t,c533=t,c534=t,c535=t,c536=t,c537=t,c538=t,c539=t,c540=t,c541=t,c542=t," - "c543=t,c544=t,c545=t,c546=t,c547=t,c548=t,c549=t,c550=t,c551=t,c552=t,c553=t,c554=t,c555=t,c556=t,c557=t,c558=t," - "c559=t,c560=t,c561=t,c562=t,c563=t,c564=t,c565=t,c566=t,c567=t,c568=t,c569=t,c570=t,c571=t,c572=t,c573=t,c574=t," - "c575=t,c576=t,c577=t,c578=t,c579=t,c580=t,c581=t,c582=t,c583=t,c584=t,c585=t,c586=t,c587=t,c588=t,c589=t,c590=t," - "c591=t,c592=t,c593=t,c594=t,c595=t,c596=t,c597=t,c598=t,c599=t,c600=t,c601=t,c602=t,c603=t,c604=t,c605=t,c606=t," - "c607=t,c608=t,c609=t,c610=t,c611=t,c612=t,c613=t,c614=t," - "c615=t,c616=t,c617=t,c618=t,c619=t,c620=t,c621=t,c622=t,c623=t,c624=t,c625=t,c626=t,c627=t,c628=t,c629=t,c630=t," - "c631=t,c632=t,c633=t,c634=t,c635=t,c636=t,c637=t,c638=t,c639=t,c640=t,c641=t,c642=t,c643=t,c644=t,c645=t,c646=t," - "c647=t,c648=t,c649=t,c650=t,c651=t,c652=t,c653=t,c654=t,c655=t,c656=t,c657=t,c658=t,c659=t,c660=t,c661=t,c662=t," - "c663=t,c664=t,c665=t,c666=t,c667=t,c668=t,c669=t,c670=t,c671=t,c672=t,c673=t,c674=t,c675=t,c676=t,c677=t,c678=t," - "c679=t,c680=t,c681=t,c682=t,c683=t,c684=t,c685=t,c686=t,c687=t,c688=t,c689=t,c690=t,c691=t,c692=t,c693=t,c694=t," - "c695=t,c696=t,c697=t,c698=t,c699=t,c700=t,c701=t,c702=t,c703=t,c704=t,c705=t,c706=t,c707=t,c708=t,c709=t,c710=t," - "c711=t,c712=t,c713=t,c714=t,c715=t,c716=t,c717=t,c718=t,c719=t,c720=t,c721=t,c722=t,c723=t,c724=t,c725=t,c726=t," - "c727=t,c728=t,c729=t,c730=t,c731=t,c732=t,c733=t,c734=t,c735=t,c736=t,c737=t,c738=t,c739=t,c740=t,c741=t,c742=t," - "c743=t,c744=t,c745=t,c746=t,c747=t,c748=t,c749=t,c750=t,c751=t,c752=t,c753=t,c754=t,c755=t,c756=t,c757=t,c758=t," - "c759=t,c760=t,c761=t,c762=t,c763=t,c764=t,c765=t,c766=t,c767=t,c768=t,c769=t,c770=t,c771=t,c772=t,c773=t,c774=t," - "c775=t,c776=t,c777=t,c778=t,c779=t,c780=t,c781=t,c782=t," - "c783=t,c784=t,c785=t,c786=t,c787=t,c788=t,c789=t,c790=t,c791=t,c792=t,c793=t,c794=t,c795=t,c796=t,c797=t,c798=t," - "c799=t,c800=t,c801=t,c802=t,c803=t,c804=t,c805=t,c806=t,c807=t,c808=t,c809=t,c810=t,c811=t,c812=t,c813=t," - "c814=t,c815=t,c816=t,c817=t,c818=t,c819=t,c820=t,c821=t,c822=t,c823=t,c824=t,c825=t,c826=t,c827=t,c828=t,c829=t," - "c830=t,c831=t,c832=t,c833=t,c834=t,c835=t,c836=t,c837=t,c838=t,c839=t,c840=t,c841=t,c842=t,c843=t,c844=t,c845=t," - "c846=t,c847=t,c848=t,c849=t,c850=t,c851=t,c852=t,c853=t,c854=t,c855=t,c856=t,c857=t,c858=t,c859=t,c860=t,c861=t," - "c862=t," - "c863=t,c864=t,c865=t,c866=t,c867=t,c868=t,c869=t,c870=t,c871=t,c872=t,c873=t,c874=t,c875=t,c876=t,c877=t,c878=t," - "c879=t,c880=t,c881=t,c882=t,c883=t,c884=t,c885=t,c886=t,c887=t,c888=t,c889=t,c890=t,c891=t,c892=t,c893=t,c894=t," - "c895=t,c896=t,c897=t,c898=t,c899=t,c900=t,c901=t,c902=t,c903=t,c904=t,c905=t,c906=t,c907=t,c908=t,c909=t,c910=t," - "c911=t,c912=t,c913=t,c914=t,c915=t,c916=t,c917=t,c918=t,c919=t,c920=t,c921=t,c922=t,c923=t,c924=t,c925=t,c926=t," - "c927=t,c928=t,c929=t,c930=t,c931=t,c932=t,c933=t,c934=t,c935=t,c936=t,c937=t,c938=t,c939=t,c940=t,c941=t,c942=t," - "c943=t,c944=t,c945=t,c946=t,c947=t,c948=t,c949=t,c950=t,c951=t,c952=t,c953=t,c954=t,c955=t,c956=t,c957=t,c958=t," - "c959=t,c960=t,c961=t,c962=t,c963=t,c964=t,c965=t,c966=t,c967=t,c968=t,c969=t,c970=t,c971=t,c972=t,c973=t,c974=t," - "c975=t,c976=t,c977=t,c978=t,c979=t,c980=t,c981=t,c982=t,c983=t,c984=t,c985=t,c986=t,c987=t,c988=t,c989=t,c990=t," - "c991=t,c992=t,c993=t,c994=t,c995=t,c996=t,c997=t,c998=t,c999=t,c1000=t,c1001=t,c1002=t,c1003=t,c1004=t,c1005=t," - "c1006=t,c1007=t,c1008=t,c1009=t,c1010=t,c1011=t,c1012=t,c1013=t,c1014=t,c1015=t,c1016=t,c1017=t,c1018=t,c1019=t," - "c1020=t,c1021=t,c1022=t,c1023=t,c1024=t,c1025=t,c1026=t," - "c1027=t,c1028=t,c1029=t,c1030=t,c1031=t,c1032=t,c1033=t,c1034=t,c1035=t,c1036=t,c1037=t,c1038=t,c1039=t,c1040=t," - "c1041=t,c1042=t,c1043=t,c1044=t,c1045=t,c1046=t,c1047=t,c1048=t,c1049=t,c1050=t,c1051=t,c1052=t,c1053=t,c1054=t," - "c1055=t,c1056=t,c1057=t,c1058=t,c1059=t,c1060=t,c1061=t,c1062=t,c1063=t,c1064=t,c1065=t,c1066=t,c1067=t,c1068=t," - "c1069=t,c1070=t,c1071=t,c1072=t,c1073=t,c1074=t,c1075=t,c1076=t,c1077=t,c1078=t,c1079=t,c1080=t,c1081=t,c1082=t," - "c1083=t,c1084=t,c1085=t,c1086=t,c1087=t,c1088=t,c1089=t,c1090=t,c1091=t,c1092=t,c1093=t,c1094=t,c1095=t,c1096=t," - "c1097=t,c1098=t,c1099=t,c1100=t,c1101=t,c1102=t,c1103=t,c1104=t,c1105=t,c1106=t,c1107=t,c1108=t,c1109=t,c1110=t," - "c1111=t,c1112=t,c1113=t,c1114=t,c1115=t,c1116=t,c1117=t,c1118=t,c1119=t,c1120=t,c1121=t,c1122=t,c1123=t,c1124=t," - "c1125=t,c1126=t,c1127=t,c1128=t,c1129=t,c1130=t,c1131=t,c1132=t,c1133=t,c1134=t,c1135=t,c1136=t,c1137=t,c1138=t," - "c1139=t,c1140=t,c1141=t,c1142=t,c1143=t,c1144=t,c1145=t,c1146=t,c1147=t,c1148=t,c1149=t,c1150=t,c1151=t,c1152=t," - "c1153=t,c1154=t,c1155=t,c1156=t,c1157=t,c1158=t,c1159=t,c1160=t,c1161=t,c1162=t,c1163=t,c1164=t,c1165=t,c1166=t," - "c1167=t,c1168=t,c1169=t,c1170=t,c1171=t,c1172=t,c1173=t," - "c1174=t,c1175=t,c1176=t,c1177=t,c1178=t,c1179=t,c1180=t,c1181=t,c1182=t,c1183=t,c1184=t,c1185=t,c1186=t,c1187=t," - "c1188=t,c1189=t,c1190=t,c1191=t,c1192=t,c1193=t,c1194=t,c1195=t,c1196=t,c1197=t,c1198=t,c1199=t,c1200=t,c1201=t," - "c1202=t,c1203=t,c1204=t,c1205=t,c1206=t,c1207=t,c1208=t,c1209=t,c1210=t,c1211=t,c1212=t,c1213=t,c1214=t,c1215=t," - "c1216=t,c1217=t,c1218=t,c1219=t,c1220=t,c1221=t,c1222=t,c1223=t,c1224=t,c1225=t,c1226=t,c1227=t,c1228=t,c1229=t," - "c1230=t,c1231=t,c1232=t,c1233=t,c1234=t,c1235=t,c1236=t,c1237=t,c1238=t,c1239=t,c1240=t,c1241=t,c1242=t,c1243=t," - "c1244=t,c1245=t,c1246=t,c1247=t,c1248=t,c1249=t,c1250=t,c1251=t,c1252=t,c1253=t,c1254=t,c1255=t,c1256=t,c1257=t," - "c1258=t,c1259=t,c1260=t,c1261=t,c1262=t,c1263=t,c1264=t,c1265=t,c1266=t,c1267=t,c1268=t,c1269=t,c1270=t,c1271=t," - "c1272=t,c1273=t,c1274=t,c1275=t,c1276=t,c1277=t,c1278=t,c1279=t,c1280=t,c1281=t,c1282=t,c1283=t,c1284=t,c1285=t," - "c1286=t,c1287=t,c1288=t,c1289=t,c1290=t,c1291=t,c1292=t,c1293=t,c1294=t,c1295=t,c1296=t,c1297=t,c1298=t,c1299=t," - "c1300=t,c1301=t,c1302=t,c1303=t,c1304=t,c1305=t,c1306=t,c1307=t,c1308=t,c1309=t,c1310=t,c1311=t,c1312=t,c1313=t," - "c1314=t,c1315=t,c1316=t,c1317=t,c1318=t,c1319=t,c1320=t," - "c1321=t,c1322=t,c1323=t,c1324=t,c1325=t,c1326=t,c1327=t,c1328=t,c1329=t,c1330=t,c1331=t,c1332=t,c1333=t,c1334=t," - "c1335=t,c1336=t,c1337=t,c1338=t,c1339=t,c1340=t,c1341=t,c1342=t,c1343=t,c1344=t,c1345=t,c1346=t,c1347=t," - "c1348=t,c1349=t,c1350=t,c1351=t,c1352=t,c1353=t,c1354=t,c1355=t,c1356=t,c1357=t,c1358=t,c1359=t,c1360=t,c1361=t," - "c1362=t,c1363=t,c1364=t,c1365=t,c1366=t,c1367=t,c1368=t,c1369=t,c1370=t,c1371=t,c1372=t,c1373=t,c1374=t,c1375=t," - "c1376=t,c1377=t,c1378=t,c1379=t,c1380=t,c1381=t,c1382=t,c1383=t,c1384=t,c1385=t,c1386=t,c1387=t,c1388=t,c1389=t," - "c1390=t,c1391=t,c1392=t,c1393=t,c1394=t,c1395=t,c1396=t,c1397=t,c1398=t,c1399=t,c1400=t,c1401=t,c1402=t,c1403=t," - "c1404=t,c1405=t,c1406=t,c1407=t,c1408=t,c1409=t,c1410=t,c1411=t,c1412=t,c1413=t,c1414=t,c1415=t,c1416=t,c1417=t," - "c1418=t,c1419=t,c1420=t,c1421=t,c1422=t,c1423=t,c1424=t,c1425=t,c1426=t,c1427=t,c1428=t,c1429=t,c1430=t,c1431=t," - "c1432=t,c1433=t,c1434=t,c1435=t,c1436=t,c1437=t,c1438=t,c1439=t,c1440=t,c1441=t,c1442=t,c1443=t,c1444=t,c1445=t," - "c1446=t,c1447=t,c1448=t,c1449=t,c1450=t,c1451=t,c1452=t,c1453=t,c1454=t,c1455=t,c1456=t,c1457=t,c1458=t,c1459=t," - "c1460=t,c1461=t,c1462=t,c1463=t,c1464=t,c1465=t,c1466=t,c1467=t,c1468=t,c1469=t,c1470=t,c1471=t,c1472=t,c1473=t," - "c1474=t,c1475=t,c1476=t,c1477=t,c1478=t,c1479=t,c1480=t,c1481=t,c1482=t,c1483=t,c1484=t,c1485=t,c1486=t,c1487=t," - "c1488=t,c1489=t,c1490=t,c1491=t,c1492=t,c1493=t,c1494=t," - "c1495=t,c1496=t,c1497=t,c1498=t,c1499=t,c1500=t,c1501=t,c1502=t,c1503=t,c1504=t,c1505=t,c1506=t,c1507=t,c1508=t," - "c1509=t,c1510=t,c1511=t,c1512=t,c1513=t,c1514=t,c1515=t,c1516=t,c1517=t,c1518=t,c1519=t,c1520=t,c1521=t,c1522=t," - "c1523=t,c1524=t,c1525=t,c1526=t,c1527=t,c1528=t,c1529=t,c1530=t,c1531=t,c1532=t,c1533=t,c1534=t,c1535=t,c1536=t," - "c1537=t,c1538=t,c1539=t,c1540=t,c1541=t,c1542=t,c1543=t,c1544=t,c1545=t,c1546=t,c1547=t,c1548=t,c1549=t,c1550=t," - "c1551=t,c1552=t,c1553=t,c1554=t,c1555=t,c1556=t,c1557=t,c1558=t,c1559=t,c1560=t,c1561=t,c1562=t,c1563=t,c1564=t," - "c1565=t,c1566=t,c1567=t,c1568=t,c1569=t,c1570=t,c1571=t,c1572=t,c1573=t,c1574=t,c1575=t,c1576=t,c1577=t,c1578=t," - "c1579=t,c1580=t,c1581=t,c1582=t,c1583=t,c1584=t,c1585=t,c1586=t,c1587=t,c1588=t,c1589=t,c1590=t,c1591=t,c1592=t," - "c1593=t,c1594=t,c1595=t,c1596=t,c1597=t,c1598=t,c1599=t,c1600=t,c1601=t,c1602=t,c1603=t,c1604=t,c1605=t,c1606=t," - "c1607=t,c1608=t,c1609=t,c1610=t,c1611=t,c1612=t,c1613=t,c1614=t,c1615=t,c1616=t,c1617=t,c1618=t,c1619=t,c1620=t," - "c1621=t,c1622=t,c1623=t,c1624=t,c1625=t,c1626=t,c1627=t,c1628=t,c1629=t,c1630=t,c1631=t,c1632=t,c1633=t,c1634=t," - "c1635=t,c1636=t,c1637=t,c1638=t,c1639=t,c1640=t,c1641=t," - "c1642=t,c1643=t,c1644=t,c1645=t,c1646=t,c1647=t,c1648=t,c1649=t,c1650=t,c1651=t,c1652=t,c1653=t,c1654=t,c1655=t," - "c1656=t,c1657=t,c1658=t,c1659=t,c1660=t,c1661=t,c1662=t,c1663=t,c1664=t,c1665=t,c1666=t,c1667=t,c1668=t,c1669=t," - "c1670=t,c1671=t,c1672=t,c1673=t,c1674=t,c1675=t,c1676=t,c1677=t,c1678=t,c1679=t,c1680=t,c1681=t,c1682=t,c1683=t," - "c1684=t,c1685=t,c1686=t,c1687=t,c1688=t,c1689=t,c1690=t,c1691=t,c1692=t,c1693=t,c1694=t,c1695=t,c1696=t,c1697=t," - "c1698=t,c1699=t,c1700=t,c1701=t,c1702=t,c1703=t,c1704=t,c1705=t,c1706=t,c1707=t,c1708=t,c1709=t,c1710=t,c1711=t," - "c1712=t,c1713=t,c1714=t,c1715=t,c1716=t,c1717=t,c1718=t,c1719=t,c1720=t,c1721=t,c1722=t,c1723=t,c1724=t,c1725=t," - "c1726=t,c1727=t,c1728=t,c1729=t,c1730=t,c1731=t,c1732=t,c1733=t,c1734=t,c1735=t,c1736=t,c1737=t,c1738=t,c1739=t," - "c1740=t,c1741=t,c1742=t,c1743=t,c1744=t,c1745=t,c1746=t,c1747=t,c1748=t,c1749=t,c1750=t,c1751=t,c1752=t,c1753=t," - "c1754=t,c1755=t,c1756=t,c1757=t,c1758=t,c1759=t,c1760=t,c1761=t,c1762=t,c1763=t,c1764=t,c1765=t,c1766=t,c1767=t," - "c1768=t,c1769=t,c1770=t,c1771=t,c1772=t,c1773=t,c1774=t,c1775=t,c1776=t,c1777=t,c1778=t,c1779=t,c1780=t,c1781=t," - "c1782=t,c1783=t,c1784=t,c1785=t,c1786=t,c1787=t,c1788=t," - "c1789=t,c1790=t,c1791=t,c1792=t,c1793=t,c1794=t,c1795=t,c1796=t,c1797=t,c1798=t,c1799=t,c1800=t,c1801=t,c1802=t," - "c1803=t,c1804=t,c1805=t,c1806=t,c1807=t,c1808=t,c1809=t,c1810=t,c1811=t,c1812=t,c1813=t,c1814=t,c1815=t," - "c1816=t,c1817=t,c1818=t,c1819=t,c1820=t,c1821=t,c1822=t,c1823=t,c1824=t,c1825=t,c1826=t,c1827=t,c1828=t,c1829=t," - "c1830=t,c1831=t,c1832=t,c1833=t,c1834=t,c1835=t,c1836=t,c1837=t,c1838=t,c1839=t,c1840=t,c1841=t,c1842=t,c1843=t," - "c1844=t,c1845=t,c1846=t,c1847=t,c1848=t,c1849=t,c1850=t,c1851=t,c1852=t,c1853=t,c1854=t,c1855=t,c1856=t,c1857=t," - "c1858=t,c1859=t,c1860=t,c1861=t,c1862=t,c1863=t,c1864=t,c1865=t,c1866=t,c1867=t,c1868=t,c1869=t,c1870=t,c1871=t," - "c1872=t,c1873=t,c1874=t,c1875=t,c1876=t,c1877=t,c1878=t,c1879=t,c1880=t,c1881=t,c1882=t,c1883=t,c1884=t,c1885=t," - "c1886=t,c1887=t,c1888=t,c1889=t,c1890=t,c1891=t,c1892=t,c1893=t,c1894=t,c1895=t,c1896=t,c1897=t,c1898=t,c1899=t," - "c1900=t,c1901=t,c1902=t,c1903=t,c1904=t,c1905=t,c1906=t,c1907=t,c1908=t,c1909=t,c1910=t,c1911=t,c1912=t,c1913=t," - "c1914=t,c1915=t,c1916=t,c1917=t,c1918=t,c1919=t,c1920=t,c1921=t,c1922=t,c1923=t,c1924=t,c1925=t,c1926=t,c1927=t," - "c1928=t,c1929=t,c1930=t,c1931=t,c1932=t,c1933=t,c1934=t,c1935=t,c1936=t,c1937=t,c1938=t,c1939=t,c1940=t,c1941=t," - "c1942=t,c1943=t,c1944=t,c1945=t,c1946=t,c1947=t,c1948=t,c1949=t,c1950=t,c1951=t,c1952=t,c1953=t,c1954=t,c1955=t," - "c1956=t,c1957=t,c1958=t,c1959=t,c1960=t,c1961=t,c1962=t," - "c1963=t,c1964=t,c1965=t,c1966=t,c1967=t,c1968=t,c1969=t,c1970=t,c1971=t,c1972=t,c1973=t,c1974=t,c1975=t,c1976=t," - "c1977=t,c1978=t,c1979=t,c1980=t,c1981=t,c1982=t,c1983=t,c1984=t,c1985=t,c1986=t,c1987=t,c1988=t,c1989=t,c1990=t," - "c1991=t,c1992=t,c1993=t,c1994=t,c1995=t,c1996=t,c1997=t,c1998=t,c1999=t,c2000=t,c2001=t,c2002=t,c2003=t,c2004=t," - "c2005=t,c2006=t,c2007=t,c2008=t,c2009=t,c2010=t,c2011=t,c2012=t,c2013=t,c2014=t,c2015=t,c2016=t,c2017=t,c2018=t," - "c2019=t,c2020=t,c2021=t,c2022=t,c2023=t,c2024=t,c2025=t,c2026=t,c2027=t,c2028=t,c2029=t,c2030=t,c2031=t,c2032=t," - "c2033=t,c2034=t,c2035=t,c2036=t,c2037=t,c2038=t,c2039=t,c2040=t,c2041=t,c2042=t,c2043=t,c2044=t,c2045=t,c2046=t," - "c2047=t,c2048=t,c2049=t,c2050=t,c2051=t,c2052=t,c2053=t,c2054=t,c2055=t,c2056=t,c2057=t,c2058=t,c2059=t,c2060=t," - "c2061=t,c2062=t,c2063=t,c2064=t,c2065=t,c2066=t,c2067=t,c2068=t,c2069=t,c2070=t,c2071=t,c2072=t,c2073=t,c2074=t," - "c2075=t,c2076=t,c2077=t,c2078=t,c2079=t,c2080=t,c2081=t,c2082=t,c2083=t,c2084=t,c2085=t,c2086=t,c2087=t,c2088=t," - "c2089=t,c2090=t,c2091=t,c2092=t,c2093=t,c2094=t,c2095=t,c2096=t,c2097=t,c2098=t,c2099=t,c2100=t,c2101=t,c2102=t," - "c2103=t,c2104=t,c2105=t,c2106=t,c2107=t,c2108=t,c2109=t," - "c2110=t,c2111=t,c2112=t,c2113=t,c2114=t,c2115=t,c2116=t,c2117=t,c2118=t,c2119=t,c2120=t,c2121=t,c2122=t,c2123=t," - "c2124=t,c2125=t,c2126=t,c2127=t,c2128=t,c2129=t,c2130=t,c2131=t,c2132=t,c2133=t,c2134=t,c2135=t,c2136=t,c2137=t," - "c2138=t,c2139=t,c2140=t,c2141=t,c2142=t,c2143=t,c2144=t,c2145=t,c2146=t,c2147=t,c2148=t,c2149=t,c2150=t,c2151=t," - "c2152=t,c2153=t,c2154=t,c2155=t,c2156=t,c2157=t,c2158=t,c2159=t,c2160=t,c2161=t,c2162=t,c2163=t,c2164=t,c2165=t," - "c2166=t,c2167=t,c2168=t,c2169=t,c2170=t,c2171=t,c2172=t,c2173=t,c2174=t,c2175=t,c2176=t,c2177=t,c2178=t,c2179=t," - "c2180=t,c2181=t,c2182=t,c2183=t,c2184=t,c2185=t,c2186=t,c2187=t,c2188=t,c2189=t,c2190=t,c2191=t,c2192=t,c2193=t," - "c2194=t,c2195=t,c2196=t,c2197=t,c2198=t,c2199=t,c2200=t,c2201=t,c2202=t,c2203=t,c2204=t,c2205=t,c2206=t,c2207=t," - "c2208=t,c2209=t,c2210=t,c2211=t,c2212=t,c2213=t,c2214=t,c2215=t,c2216=t,c2217=t,c2218=t,c2219=t,c2220=t,c2221=t," - "c2222=t,c2223=t,c2224=t,c2225=t,c2226=t,c2227=t,c2228=t,c2229=t,c2230=t,c2231=t,c2232=t,c2233=t,c2234=t,c2235=t," - "c2236=t,c2237=t,c2238=t,c2239=t,c2240=t,c2241=t,c2242=t,c2243=t,c2244=t,c2245=t,c2246=t,c2247=t,c2248=t,c2249=t," - "c2250=t,c2251=t,c2252=t,c2253=t,c2254=t,c2255=t,c2256=t," - "c2257=t,c2258=t,c2259=t,c2260=t,c2261=t,c2262=t,c2263=t,c2264=t,c2265=t,c2266=t,c2267=t,c2268=t,c2269=t,c2270=t," - "c2271=t,c2272=t,c2273=t,c2274=t,c2275=t,c2276=t,c2277=t,c2278=t,c2279=t,c2280=t,c2281=t,c2282=t,c2283=t," - "c2284=t,c2285=t,c2286=t,c2287=t,c2288=t,c2289=t,c2290=t,c2291=t,c2292=t,c2293=t,c2294=t,c2295=t,c2296=t,c2297=t," - "c2298=t,c2299=t,c2300=t,c2301=t,c2302=t,c2303=t,c2304=t,c2305=t,c2306=t,c2307=t,c2308=t,c2309=t,c2310=t,c2311=t," - "c2312=t,c2313=t,c2314=t,c2315=t,c2316=t,c2317=t,c2318=t,c2319=t,c2320=t,c2321=t,c2322=t,c2323=t,c2324=t,c2325=t," - "c2326=t,c2327=t,c2328=t,c2329=t,c2330=t,c2331=t,c2332=t,c2333=t,c2334=t,c2335=t,c2336=t,c2337=t,c2338=t,c2339=t," - "c2340=t,c2341=t,c2342=t,c2343=t,c2344=t,c2345=t,c2346=t,c2347=t,c2348=t,c2349=t,c2350=t,c2351=t,c2352=t,c2353=t," - "c2354=t,c2355=t,c2356=t,c2357=t,c2358=t,c2359=t,c2360=t,c2361=t,c2362=t,c2363=t,c2364=t,c2365=t,c2366=t,c2367=t," - "c2368=t,c2369=t,c2370=t,c2371=t,c2372=t,c2373=t,c2374=t,c2375=t,c2376=t,c2377=t,c2378=t,c2379=t,c2380=t,c2381=t," - "c2382=t,c2383=t,c2384=t,c2385=t,c2386=t,c2387=t,c2388=t,c2389=t,c2390=t,c2391=t,c2392=t,c2393=t,c2394=t,c2395=t," - "c2396=t,c2397=t,c2398=t,c2399=t,c2400=t,c2401=t,c2402=t,c2403=t,c2404=t,c2405=t,c2406=t,c2407=t,c2408=t,c2409=t," - "c2410=t,c2411=t,c2412=t,c2413=t,c2414=t,c2415=t,c2416=t,c2417=t,c2418=t,c2419=t,c2420=t,c2421=t,c2422=t,c2423=t," - "c2424=t,c2425=t,c2426=t,c2427=t,c2428=t,c2429=t,c2430=t," - "c2431=t,c2432=t,c2433=t,c2434=t,c2435=t,c2436=t,c2437=t,c2438=t,c2439=t,c2440=t,c2441=t,c2442=t,c2443=t,c2444=t," - "c2445=t,c2446=t,c2447=t,c2448=t,c2449=t,c2450=t,c2451=t,c2452=t,c2453=t,c2454=t,c2455=t,c2456=t,c2457=t,c2458=t," - "c2459=t,c2460=t,c2461=t,c2462=t,c2463=t,c2464=t,c2465=t,c2466=t,c2467=t,c2468=t,c2469=t,c2470=t,c2471=t,c2472=t," - "c2473=t,c2474=t,c2475=t,c2476=t,c2477=t,c2478=t,c2479=t,c2480=t,c2481=t,c2482=t,c2483=t,c2484=t,c2485=t,c2486=t," - "c2487=t,c2488=t,c2489=t,c2490=t,c2491=t,c2492=t,c2493=t,c2494=t,c2495=t,c2496=t,c2497=t,c2498=t,c2499=t,c2500=t," - "c2501=t,c2502=t,c2503=t,c2504=t,c2505=t,c2506=t,c2507=t,c2508=t,c2509=t,c2510=t,c2511=t,c2512=t,c2513=t,c2514=t," - "c2515=t,c2516=t,c2517=t,c2518=t,c2519=t,c2520=t,c2521=t,c2522=t,c2523=t,c2524=t,c2525=t,c2526=t,c2527=t,c2528=t," - "c2529=t,c2530=t,c2531=t,c2532=t,c2533=t,c2534=t,c2535=t,c2536=t,c2537=t,c2538=t,c2539=t,c2540=t,c2541=t,c2542=t," - "c2543=t,c2544=t,c2545=t,c2546=t,c2547=t,c2548=t,c2549=t,c2550=t,c2551=t,c2552=t,c2553=t,c2554=t,c2555=t,c2556=t," - "c2557=t,c2558=t,c2559=t,c2560=t,c2561=t,c2562=t,c2563=t,c2564=t,c2565=t,c2566=t,c2567=t,c2568=t,c2569=t,c2570=t," - "c2571=t,c2572=t,c2573=t,c2574=t,c2575=t,c2576=t,c2577=t," - "c2578=t,c2579=t,c2580=t,c2581=t,c2582=t,c2583=t,c2584=t,c2585=t,c2586=t,c2587=t,c2588=t,c2589=t,c2590=t,c2591=t," - "c2592=t,c2593=t,c2594=t,c2595=t,c2596=t,c2597=t,c2598=t,c2599=t,c2600=t,c2601=t,c2602=t,c2603=t,c2604=t,c2605=t," - "c2606=t,c2607=t,c2608=t,c2609=t,c2610=t,c2611=t,c2612=t,c2613=t,c2614=t,c2615=t,c2616=t,c2617=t,c2618=t,c2619=t," - "c2620=t,c2621=t,c2622=t,c2623=t,c2624=t,c2625=t,c2626=t,c2627=t,c2628=t,c2629=t,c2630=t,c2631=t,c2632=t,c2633=t," - "c2634=t,c2635=t,c2636=t,c2637=t,c2638=t,c2639=t,c2640=t,c2641=t,c2642=t,c2643=t,c2644=t,c2645=t,c2646=t,c2647=t," - "c2648=t,c2649=t,c2650=t,c2651=t,c2652=t,c2653=t,c2654=t,c2655=t,c2656=t,c2657=t,c2658=t,c2659=t,c2660=t,c2661=t," - "c2662=t,c2663=t,c2664=t,c2665=t,c2666=t,c2667=t,c2668=t,c2669=t,c2670=t,c2671=t,c2672=t,c2673=t,c2674=t,c2675=t," - "c2676=t,c2677=t,c2678=t,c2679=t,c2680=t,c2681=t,c2682=t,c2683=t,c2684=t,c2685=t,c2686=t,c2687=t,c2688=t,c2689=t," - "c2690=t,c2691=t,c2692=t,c2693=t,c2694=t,c2695=t,c2696=t,c2697=t,c2698=t,c2699=t,c2700=t,c2701=t,c2702=t,c2703=t," - "c2704=t,c2705=t,c2706=t,c2707=t,c2708=t,c2709=t,c2710=t,c2711=t,c2712=t,c2713=t,c2714=t,c2715=t,c2716=t,c2717=t," - "c2718=t,c2719=t,c2720=t,c2721=t,c2722=t,c2723=t,c2724=t," - "c2725=t,c2726=t,c2727=t,c2728=t,c2729=t,c2730=t,c2731=t,c2732=t,c2733=t,c2734=t,c2735=t,c2736=t,c2737=t,c2738=t," - "c2739=t,c2740=t,c2741=t,c2742=t,c2743=t,c2744=t,c2745=t,c2746=t,c2747=t,c2748=t,c2749=t,c2750=t,c2751=t,c2752=t," - "c2753=t,c2754=t,c2755=t,c2756=t,c2757=t,c2758=t,c2759=t,c2760=t,c2761=t,c2762=t,c2763=t,c2764=t,c2765=t,c2766=t," - "c2767=t,c2768=t,c2769=t,c2770=t,c2771=t,c2772=t,c2773=t,c2774=t,c2775=t,c2776=t,c2777=t,c2778=t,c2779=t,c2780=t," - "c2781=t,c2782=t,c2783=t,c2784=t,c2785=t,c2786=t,c2787=t,c2788=t,c2789=t,c2790=t,c2791=t,c2792=t,c2793=t,c2794=t," - "c2795=t,c2796=t,c2797=t,c2798=t,c2799=t,c2800=t,c2801=t,c2802=t,c2803=t,c2804=t,c2805=t,c2806=t,c2807=t,c2808=t," - "c2809=t,c2810=t,c2811=t,c2812=t,c2813=t,c2814=t,c2815=t,c2816=t,c2817=t,c2818=t,c2819=t,c2820=t,c2821=t,c2822=t," - "c2823=t,c2824=t,c2825=t,c2826=t,c2827=t,c2828=t,c2829=t,c2830=t,c2831=t,c2832=t,c2833=t,c2834=t,c2835=t,c2836=t," - "c2837=t,c2838=t,c2839=t,c2840=t,c2841=t,c2842=t,c2843=t,c2844=t,c2845=t,c2846=t,c2847=t,c2848=t,c2849=t,c2850=t," - "c2851=t,c2852=t,c2853=t,c2854=t,c2855=t,c2856=t,c2857=t,c2858=t,c2859=t,c2860=t,c2861=t,c2862=t,c2863=t,c2864=t," - "c2865=t,c2866=t,c2867=t,c2868=t,c2869=t,c2870=t,c2871=t," - "c2872=t,c2873=t,c2874=t,c2875=t,c2876=t,c2877=t,c2878=t,c2879=t,c2880=t,c2881=t,c2882=t,c2883=t,c2884=t,c2885=t," - "c2886=t,c2887=t,c2888=t,c2889=t,c2890=t,c2891=t,c2892=t,c2893=t,c2894=t,c2895=t,c2896=t,c2897=t,c2898=t,c2899=t," - "c2900=t,c2901=t,c2902=t,c2903=t,c2904=t,c2905=t,c2906=t,c2907=t,c2908=t,c2909=t,c2910=t,c2911=t,c2912=t,c2913=t," - "c2914=t,c2915=t,c2916=t,c2917=t,c2918=t,c2919=t,c2920=t,c2921=t,c2922=t,c2923=t,c2924=t,c2925=t,c2926=t,c2927=t," - "c2928=t,c2929=t,c2930=t,c2931=t,c2932=t,c2933=t,c2934=t,c2935=t,c2936=t,c2937=t,c2938=t,c2939=t,c2940=t,c2941=t," - "c2942=t,c2943=t,c2944=t,c2945=t,c2946=t,c2947=t,c2948=t,c2949=t,c2950=t,c2951=t,c2952=t,c2953=t,c2954=t,c2955=t," - "c2956=t,c2957=t,c2958=t,c2959=t,c2960=t,c2961=t,c2962=t,c2963=t,c2964=t,c2965=t,c2966=t,c2967=t,c2968=t,c2969=t," - "c2970=t,c2971=t,c2972=t,c2973=t,c2974=t,c2975=t,c2976=t,c2977=t,c2978=t,c2979=t,c2980=t,c2981=t,c2982=t,c2983=t," - "c2984=t,c2985=t,c2986=t,c2987=t,c2988=t,c2989=t,c2990=t,c2991=t,c2992=t,c2993=t,c2994=t,c2995=t,c2996=t,c2997=t," - "c2998=t,c2999=t,c3000=t,c3001=t,c3002=t,c3003=t,c3004=t,c3005=t,c3006=t,c3007=t,c3008=t,c3009=t,c3010=t,c3011=t," - "c3012=t,c3013=t,c3014=t,c3015=t,c3016=t,c3017=t,c3018=t," - "c3019=t,c3020=t,c3021=t,c3022=t,c3023=t,c3024=t,c3025=t,c3026=t,c3027=t,c3028=t,c3029=t,c3030=t,c3031=t,c3032=t," - "c3033=t,c3034=t,c3035=t,c3036=t,c3037=t,c3038=t,c3039=t,c3040=t,c3041=t,c3042=t,c3043=t,c3044=t,c3045=t,c3046=t," - "c3047=t,c3048=t,c3049=t,c3050=t,c3051=t,c3052=t,c3053=t,c3054=t,c3055=t,c3056=t,c3057=t,c3058=t,c3059=t,c3060=t," - "c3061=t,c3062=t,c3063=t,c3064=t,c3065=t,c3066=t,c3067=t,c3068=t,c3069=t,c3070=t,c3071=t,c3072=t,c3073=t,c3074=t," - "c3075=t,c3076=t,c3077=t,c3078=t,c3079=t,c3080=t,c3081=t,c3082=t,c3083=t,c3084=t,c3085=t,c3086=t,c3087=t,c3088=t," - "c3089=t,c3090=t,c3091=t,c3092=t,c3093=t,c3094=t,c3095=t,c3096=t,c3097=t,c3098=t,c3099=t,c3100=t,c3101=t,c3102=t," - "c3103=t,c3104=t,c3105=t,c3106=t,c3107=t,c3108=t,c3109=t,c3110=t,c3111=t,c3112=t,c3113=t,c3114=t,c3115=t,c3116=t," - "c3117=t,c3118=t,c3119=t,c3120=t,c3121=t,c3122=t,c3123=t,c3124=t,c3125=t,c3126=t,c3127=t,c3128=t,c3129=t,c3130=t," - "c3131=t,c3132=t,c3133=t,c3134=t,c3135=t,c3136=t,c3137=t,c3138=t,c3139=t,c3140=t,c3141=t,c3142=t,c3143=t,c3144=t," - "c3145=t,c3146=t,c3147=t,c3148=t,c3149=t,c3150=t,c3151=t,c3152=t,c3153=t,c3154=t,c3155=t,c3156=t,c3157=t,c3158=t," - "c3159=t,c3160=t,c3161=t,c3162=t,c3163=t,c3164=t,c3165=t," - "c3166=t,c3167=t,c3168=t,c3169=t,c3170=t,c3171=t,c3172=t,c3173=t,c3174=t,c3175=t,c3176=t,c3177=t,c3178=t,c3179=t," - "c3180=t,c3181=t,c3182=t,c3183=t,c3184=t,c3185=t,c3186=t,c3187=t,c3188=t,c3189=t,c3190=t,c3191=t,c3192=t,c3193=t," - "c3194=t,c3195=t,c3196=t,c3197=t,c3198=t,c3199=t,c3200=t,c3201=t,c3202=t,c3203=t,c3204=t,c3205=t,c3206=t,c3207=t," - "c3208=t,c3209=t,c3210=t,c3211=t,c3212=t,c3213=t,c3214=t,c3215=t,c3216=t,c3217=t,c3218=t,c3219=t,c3220=t,c3221=t," - "c3222=t,c3223=t,c3224=t,c3225=t,c3226=t,c3227=t,c3228=t,c3229=t,c3230=t,c3231=t,c3232=t,c3233=t,c3234=t,c3235=t," - "c3236=t,c3237=t,c3238=t,c3239=t,c3240=t,c3241=t,c3242=t,c3243=t,c3244=t,c3245=t,c3246=t,c3247=t,c3248=t,c3249=t," - "c3250=t,c3251=t,c3252=t,c3253=t,c3254=t,c3255=t,c3256=t,c3257=t,c3258=t,c3259=t,c3260=t,c3261=t,c3262=t,c3263=t," - "c3264=t,c3265=t,c3266=t,c3267=t,c3268=t,c3269=t,c3270=t,c3271=t,c3272=t,c3273=t,c3274=t,c3275=t,c3276=t,c3277=t," - "c3278=t,c3279=t,c3280=t,c3281=t,c3282=t,c3283=t,c3284=t,c3285=t,c3286=t,c3287=t,c3288=t,c3289=t,c3290=t,c3291=t," - "c3292=t,c3293=t,c3294=t,c3295=t,c3296=t,c3297=t,c3298=t,c3299=t,c3300=t,c3301=t,c3302=t,c3303=t,c3304=t,c3305=t," - "c3306=t,c3307=t,c3308=t,c3309=t,c3310=t,c3311=t,c3312=t," - "c3313=t,c3314=t,c3315=t,c3316=t,c3317=t,c3318=t,c3319=t,c3320=t,c3321=t,c3322=t,c3323=t,c3324=t,c3325=t,c3326=t," - "c3327=t,c3328=t,c3329=t,c3330=t,c3331=t,c3332=t,c3333=t,c3334=t,c3335=t,c3336=t,c3337=t,c3338=t,c3339=t,c3340=t," - "c3341=t,c3342=t,c3343=t,c3344=t,c3345=t,c3346=t,c3347=t,c3348=t,c3349=t,c3350=t,c3351=t,c3352=t,c3353=t,c3354=t," - "c3355=t,c3356=t,c3357=t,c3358=t,c3359=t,c3360=t,c3361=t,c3362=t,c3363=t,c3364=t,c3365=t,c3366=t,c3367=t,c3368=t," - "c3369=t,c3370=t,c3371=t,c3372=t,c3373=t,c3374=t,c3375=t,c3376=t,c3377=t,c3378=t,c3379=t,c3380=t,c3381=t,c3382=t," - "c3383=t,c3384=t,c3385=t,c3386=t,c3387=t,c3388=t,c3389=t,c3390=t,c3391=t,c3392=t,c3393=t,c3394=t,c3395=t,c3396=t," - "c3397=t,c3398=t,c3399=t,c3400=t,c3401=t,c3402=t,c3403=t,c3404=t,c3405=t,c3406=t,c3407=t,c3408=t,c3409=t,c3410=t," - "c3411=t,c3412=t,c3413=t,c3414=t,c3415=t,c3416=t,c3417=t,c3418=t,c3419=t,c3420=t,c3421=t,c3422=t,c3423=t,c3424=t," - "c3425=t,c3426=t,c3427=t,c3428=t,c3429=t,c3430=t,c3431=t,c3432=t,c3433=t,c3434=t,c3435=t,c3436=t,c3437=t,c3438=t," - "c3439=t,c3440=t,c3441=t,c3442=t,c3443=t,c3444=t,c3445=t,c3446=t,c3447=t,c3448=t,c3449=t,c3450=t,c3451=t,c3452=t," - "c3453=t,c3454=t,c3455=t,c3456=t,c3457=t,c3458=t,c3459=t," - "c3460=t,c3461=t,c3462=t,c3463=t,c3464=t,c3465=t,c3466=t,c3467=t,c3468=t,c3469=t,c3470=t,c3471=t,c3472=t,c3473=t," - "c3474=t,c3475=t,c3476=t,c3477=t,c3478=t,c3479=t,c3480=t,c3481=t,c3482=t,c3483=t,c3484=t,c3485=t,c3486=t,c3487=t," - "c3488=t,c3489=t,c3490=t,c3491=t,c3492=t,c3493=t,c3494=t,c3495=t,c3496=t,c3497=t,c3498=t,c3499=t,c3500=t,c3501=t," - "c3502=t,c3503=t,c3504=t,c3505=t,c3506=t,c3507=t,c3508=t,c3509=t,c3510=t,c3511=t,c3512=t,c3513=t," - "c3514=t,c3515=t,c3516=t,c3517=t,c3518=t,c3519=t,c3520=t,c3521=t,c3522=t,c3523=t,c3524=t,c3525=t,c3526=t,c3527=t," - "c3528=t,c3529=t,c3530=t,c3531=t,c3532=t,c3533=t,c3534=t,c3535=t,c3536=t,c3537=t,c3538=t,c3539=t,c3540=t,c3541=t," - "c3542=t,c3543=t,c3544=t,c3545=t,c3546=t,c3547=t,c3548=t,c3549=t,c3550=t,c3551=t,c3552=t,c3553=t,c3554=t,c3555=t," - "c3556=t,c3557=t,c3558=t,c3559=t,c3560=t,c3561=t,c3562=t,c3563=t,c3564=t,c3565=t,c3566=t,c3567=t,c3568=t,c3569=t," - "c3570=t,c3571=t,c3572=t,c3573=t,c3574=t,c3575=t,c3576=t,c3577=t,c3578=t,c3579=t,c3580=t,c3581=t,c3582=t,c3583=t," - "c3584=t,c3585=t,c3586=t,c3587=t,c3588=t,c3589=t,c3590=t,c3591=t,c3592=t,c3593=t,c3594=t,c3595=t,c3596=t,c3597=t," - "c3598=t,c3599=t,c3600=t,c3601=t,c3602=t,c3603=t,c3604=t,c3605=t,c3606=t,c3607=t,c3608=t,c3609=t,c3610=t,c3611=t," - "c3612=t,c3613=t,c3614=t,c3615=t,c3616=t,c3617=t,c3618=t,c3619=t,c3620=t,c3621=t,c3622=t,c3623=t,c3624=t,c3625=t," - "c3626=t,c3627=t,c3628=t,c3629=t,c3630=t,c3631=t,c3632=t,c3633=t,c3634=t,c3635=t,c3636=t,c3637=t,c3638=t,c3639=t," - "c3640=t,c3641=t,c3642=t,c3643=t,c3644=t,c3645=t,c3646=t,c3647=t,c3648=t,c3649=t,c3650=t,c3651=t,c3652=t,c3653=t," - "c3654=t,c3655=t,c3656=t,c3657=t,c3658=t,c3659=t,c3660=t," - "c3661=t,c3662=t,c3663=t,c3664=t,c3665=t,c3666=t,c3667=t,c3668=t,c3669=t,c3670=t,c3671=t,c3672=t,c3673=t,c3674=t," - "c3675=t,c3676=t,c3677=t,c3678=t,c3679=t,c3680=t,c3681=t,c3682=t,c3683=t,c3684=t,c3685=t,c3686=t,c3687=t,c3688=t," - "c3689=t,c3690=t,c3691=t,c3692=t,c3693=t,c3694=t,c3695=t,c3696=t,c3697=t,c3698=t,c3699=t,c3700=t,c3701=t,c3702=t," - "c3703=t,c3704=t,c3705=t,c3706=t,c3707=t,c3708=t,c3709=t,c3710=t,c3711=t,c3712=t,c3713=t,c3714=t,c3715=t,c3716=t," - "c3717=t,c3718=t,c3719=t,c3720=t,c3721=t,c3722=t,c3723=t,c3724=t,c3725=t,c3726=t,c3727=t,c3728=t,c3729=t,c3730=t," - "c3731=t,c3732=t,c3733=t,c3734=t,c3735=t,c3736=t,c3737=t,c3738=t,c3739=t,c3740=t,c3741=t,c3742=t,c3743=t,c3744=t," - "c3745=t,c3746=t,c3747=t,c3748=t,c3749=t,c3750=t,c3751=t,c3752=t,c3753=t,c3754=t,c3755=t,c3756=t,c3757=t,c3758=t," - "c3759=t,c3760=t,c3761=t,c3762=t,c3763=t,c3764=t,c3765=t,c3766=t,c3767=t,c3768=t,c3769=t,c3770=t,c3771=t,c3772=t," - "c3773=t,c3774=t,c3775=t,c3776=t,c3777=t,c3778=t,c3779=t,c3780=t,c3781=t,c3782=t,c3783=t,c3784=t,c3785=t,c3786=t," - "c3787=t,c3788=t,c3789=t,c3790=t,c3791=t,c3792=t,c3793=t,c3794=t,c3795=t,c3796=t,c3797=t,c3798=t,c3799=t,c3800=t," - "c3801=t,c3802=t,c3803=t,c3804=t,c3805=t,c3806=t,c3807=t," - "c3808=t,c3809=t,c3810=t,c3811=t,c3812=t,c3813=t,c3814=t,c3815=t,c3816=t,c3817=t,c3818=t,c3819=t,c3820=t,c3821=t," - "c3822=t,c3823=t,c3824=t,c3825=t,c3826=t,c3827=t,c3828=t,c3829=t,c3830=t,c3831=t,c3832=t,c3833=t,c3834=t,c3835=t," - "c3836=t,c3837=t,c3838=t,c3839=t,c3840=t,c3841=t,c3842=t,c3843=t,c3844=t,c3845=t,c3846=t,c3847=t,c3848=t,c3849=t," - "c3850=t,c3851=t,c3852=t,c3853=t,c3854=t,c3855=t,c3856=t,c3857=t,c3858=t,c3859=t,c3860=t,c3861=t,c3862=t,c3863=t," - "c3864=t,c3865=t,c3866=t,c3867=t,c3868=t,c3869=t,c3870=t,c3871=t,c3872=t,c3873=t,c3874=t,c3875=t,c3876=t,c3877=t," - "c3878=t,c3879=t,c3880=t,c3881=t,c3882=t,c3883=t,c3884=t,c3885=t,c3886=t,c3887=t,c3888=t,c3889=t,c3890=t,c3891=t," - "c3892=t,c3893=t,c3894=t,c3895=t,c3896=t,c3897=t,c3898=t,c3899=t,c3900=t,c3901=t,c3902=t,c3903=t,c3904=t,c3905=t," - "c3906=t,c3907=t,c3908=t,c3909=t,c3910=t,c3911=t,c3912=t,c3913=t,c3914=t,c3915=t,c3916=t,c3917=t,c3918=t,c3919=t," - "c3920=t,c3921=t,c3922=t,c3923=t,c3924=t,c3925=t,c3926=t,c3927=t,c3928=t,c3929=t,c3930=t,c3931=t,c3932=t,c3933=t," - "c3934=t,c3935=t,c3936=t,c3937=t,c3938=t,c3939=t,c3940=t,c3941=t,c3942=t,c3943=t,c3944=t,c3945=t,c3946=t,c3947=t," - "c3948=t,c3949=t,c3950=t,c3951=t,c3952=t,c3953=t,c3954=t," - "c3955=t,c3956=t,c3957=t,c3958=t,c3959=t,c3960=t,c3961=t,c3962=t,c3963=t,c3964=t,c3965=t,c3966=t,c3967=t,c3968=t," - "c3969=t,c3970=t,c3971=t,c3972=t,c3973=t,c3974=t,c3975=t,c3976=t,c3977=t,c3978=t,c3979=t,c3980=t,c3981=t,c3982=t," - "c3983=t,c3984=t,c3985=t,c3986=t,c3987=t,c3988=t,c3989=t,c3990=t,c3991=t,c3992=t,c3993=t,c3994=t,c3995=t,c3996=t," - "c3997=t,c3998=t,c3999=t,c4000=t,c4001=t,c4002=t,c4003=t,c4004=t,c4005=t,c4006=t,c4007=t,c4008=t,c4009=t,c4010=t," - "c4011=t,c4012=t,c4013=t,c4014=t,c4015=t,c4016=t,c4017=t,c4018=t,c4019=t,c4020=t,c4021=t,c4022=t,c4023=t,c4024=t," - "c4025=t,c4026=t,c4027=t,c4028=t,c4029=t,c4030=t,c4031=t,c4032=t,c4033=t,c4034=t,c4035=t,c4036=t,c4037=t,c4038=t," - "c4039=t,c4040=t,c4041=t,c4042=t,c4043=t,c4044=t,c4045=t,c4046=t,c4047=t,c4048=t,c4049=t,c4050=t,c4051=t,c4052=t," - "c4053=t,c4054=t,c4055=t,c4056=t,c4057=t,c4058=t,c4059=t,c4060=t,c4061=t,c4062=t,c4063=t,c4064=t,c4065=t,c4066=t," - "c4067=t,c4068=t,c4069=t,c4070=t,c4071=t,c4072=t,c4073=t,c4074=t,c4075=t,c4076=t,c4077=t,c4078=t,c4079=t,c4080=t," - "c4081=t,c4082=t,c4083=t,c4084=t,c4085=t,c4086=t,c4087=t,c4088=t,c4089=t,c4090=t,c4091=t,c4092=t,c4093=t " - "1626006833640000000"}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} +} \ No newline at end of file diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a4a185e5ee0895a17ec04a337512a3e45f58b33c..fe6e392ead429ab55a1282b14860fa3893e3e70f 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 32 +#define MALLOC_ALIGN_BYTES 32 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); @@ -38,7 +38,8 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; } else { - return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows); + return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + + BitmapLen(numOfRows); } } @@ -279,7 +280,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); } pColumnInfoData->varmeta.length = len + oldLen; @@ -358,7 +359,11 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSiz size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { - if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { + if (pDataBlock->info.rows > 0) { +// ASSERT(pDataBlock->info.dataLoad == 1); + } + + if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) { return 0; } @@ -1137,14 +1142,15 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF } void blockDataCleanup(SSDataBlock* pDataBlock) { + blockDataEmpty(pDataBlock); SDataBlockInfo* pInfo = &pDataBlock->info; - - pInfo->rows = 0; pInfo->id.uid = 0; pInfo->id.groupId = 0; - pInfo->window.ekey = 0; - pInfo->window.skey = 0; +} +void blockDataEmpty(SSDataBlock* pDataBlock) { + SDataBlockInfo* pInfo = &pDataBlock->info; + ASSERT(pInfo->rows <= pDataBlock->info.capacity); if (pInfo->capacity == 0) { return; } @@ -1154,11 +1160,18 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfoDataCleanup(p, pInfo->capacity); } + + pInfo->rows = 0; + pInfo->dataLoad = 0; + pInfo->window.ekey = 0; + pInfo->window.skey = 0; } // todo temporarily disable it + static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { - ASSERT(numOfRows > 0 /*&& pBlockInfo->capacity >= pBlockInfo->rows*/); + ASSERT(numOfRows > 0); + if (numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } @@ -1186,7 +1199,6 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); - ASSERT(pColumn->info.bytes); // make sure the allocated memory is MALLOC_ALIGN_BYTES aligned @@ -1202,6 +1214,12 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } pColumn->pData = tmp; + + // todo remove it soon +#if defined LINUX + ASSERT((((uint64_t)pColumn->pData) & (MALLOC_ALIGN_BYTES - 1)) == 0x0); +#endif + if (clearPayload) { memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); } @@ -1210,7 +1228,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_SUCCESS; } -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1249,6 +1267,25 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { return TSDB_CODE_SUCCESS; } +int32_t blockDataEnsureCapacityNoClear(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + if (numOfRows == 0 || numOfRows <= pDataBlock->info.capacity) { + return TSDB_CODE_SUCCESS; + } + + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false); + if (code) { + return code; + } + } + + pDataBlock->info.capacity = numOfRows; + return TSDB_CODE_SUCCESS; +} + void blockDataFreeRes(SSDataBlock* pBlock) { int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1621,6 +1658,8 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total); + + // clear the offset value of the unused entries. memset(&pColInfoData->varmeta.offset[total - n], 0, n); } else { int32_t bytes = pColInfoData->info.bytes; @@ -1635,7 +1674,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { } if (pBlock->info.rows <= n) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); } else { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { @@ -1652,12 +1691,22 @@ static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_ if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n); memset(&pColInfoData->varmeta.offset[n], 0, total - n); + } else { // reset the bitmap value + /*int32_t stopIndex = BitmapLen(n) * 8; + for(int32_t i = n; i < stopIndex; ++i) { + colDataClearNull_f(pColInfoData->nullbitmap, i); + } + + int32_t remain = BitmapLen(total) - BitmapLen(n); + if (remain > 0) { + memset(pColInfoData->nullbitmap+BitmapLen(n), 0, remain); + }*/ } } int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) { if (n == 0) { - blockDataCleanup(pBlock); + blockDataEmpty(pBlock); return TSDB_CODE_SUCCESS; } @@ -1905,9 +1954,9 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "\n", + "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, - pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version); + pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { @@ -2009,6 +2058,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param suid * */ +#if 0 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { int32_t bufSize = sizeof(SSubmitReq); @@ -2174,6 +2224,167 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } +#endif + +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + int32_t sz = 1; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + } + + for (int32_t i = 0; i < sz; ++i) { + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + + if (colNum <= 1) { // invalid if only with TS col + continue; + } + + // the rsma result should has the same column number with schema. + ASSERT(colNum == pTSchema->numOfCols); + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; + + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + + taosArrayClear(pVals); + + bool isStartKey = false; + int32_t offset = 0; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + const STColumn* pCol = &pTSchema->columns[k]; + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + ASSERT(pColInfoData->info.type == pCol->type); + if (!isStartKey) { + isStartKey = true; + ASSERT(PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(TSKEY*)var}); + taosArrayPush(pVals, &cv); + } else if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(int64_t*)var}); + taosArrayPush(pVals, &cv); + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); + } else { + SValue sv; + if (pCol->type == pColInfoData->info.type) { + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + } else { + /** + * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation + * 2. below conversion may lead to overflow or loss, the app should select the right data type. + */ + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + } + SColVal cv = COL_VAL_VALUE(pCol->colId, pColInfoData->info.type, sv); + taosArrayPush(pVals, &cv); + } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); + } + break; + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + return TSDB_CODE_FAILED; + } + *ppReq = pReq; + return TSDB_CODE_SUCCESS; +} char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { ASSERT(stbFullName[0] != 0); @@ -2182,24 +2393,15 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); - return NULL; - } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2211,7 +2413,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); ASSERT(rname.ctbShortName && rname.ctbShortName[0]); @@ -2386,6 +2587,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += colLen[i]; } + pBlock->info.dataLoad = 1; pBlock->info.rows = numOfRows; ASSERT(pStart - pData == dataLen); return pStart; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 803d63aac9b8c6dcc2cba3f07039bd6a80e89155..11bd340f8e51e30c44eb5e80872f245c6013ce57 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -63,9 +63,9 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) +#define BIT_FLG_NONE ((uint8_t)0x0) +#define BIT_FLG_NULL ((uint8_t)0x1) +#define BIT_FLG_VALUE ((uint8_t)0x2) #pragma pack(push, 1) typedef struct { @@ -97,7 +97,7 @@ typedef struct { } \ } while (0) -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t code = 0; ASSERT(TARRAY_SIZE(aColVal) > 0); @@ -105,18 +105,18 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - SRow *pRow = NULL; - SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = TARRAY_SIZE(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; + SRow *pRow = NULL; + SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = TARRAY_SIZE(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; + int32_t iTColumn = 1; + const STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { @@ -309,12 +309,20 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { break; } + if (pb) { + if (flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + memset(pb, 0, BIT2_SIZE(pTSchema->numOfCols - 1)); + } else { + memset(pb, 0, BIT1_SIZE(pTSchema->numOfCols - 1)); + } + } + // build impl while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -327,24 +335,24 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SRow **ppRow) { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NULL); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } @@ -459,7 +467,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { } else { uint8_t *pf; uint8_t *pv; - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; switch (pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -487,10 +495,10 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); return; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); return; } @@ -537,7 +545,7 @@ static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, aIter = taosMemoryCalloc(nRow, sizeof(SRowIter *)); if (aIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -551,7 +559,7 @@ static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, // merge aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); if (aColVal == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -615,6 +623,7 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { if (iEnd - iStart > 1) { code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); + if (code) return code; } // the array is also changing, so the iStart just ++ instead of iEnd @@ -651,7 +660,7 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { SRowIter *pIter = taosMemoryCalloc(1, sizeof(*pIter)); if (pIter == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -789,7 +798,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { switch (pIter->pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -810,10 +819,10 @@ SColVal *tRowIterNext(SRowIter *pIter) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -947,11 +956,11 @@ static int32_t tRowAppendTupleToColData(SRow *pRow, STSchema *pTSchema, SColData break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); if (code) goto _exit; goto _continue; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; goto _continue; @@ -1029,17 +1038,22 @@ static int32_t tRowAppendKVToColData(SRow *pRow, STSchema *pTSchema, SColData *a } int16_t cid; - pData += tGetI16(pData, &cid); + pData += tGetI16v(pData, &cid); if (TABS(cid) == pTColumn->colId) { if (cid < 0) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { - uint32_t nData; - pData += tGetU32v(pData, &nData); - code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); - if (code) goto _exit; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint32_t nData; + pData += tGetU32v(pData, &nData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, 0); + if (code) goto _exit; + } } iCol++; goto _continue; @@ -1606,7 +1620,7 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1638,7 +1652,7 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_NULL; pColData->nVal++; @@ -1653,7 +1667,7 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; @@ -1681,7 +1695,7 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->flag |= HAS_NONE; pColData->nVal++; @@ -1702,9 +1716,9 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); } - SET_BIT2(pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pBitMap, pColData->nVal, 2); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1731,7 +1745,7 @@ static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->nVal++; return code; @@ -1742,7 +1756,7 @@ static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->nVal++; return code; @@ -1758,7 +1772,7 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); return tColDataPutValue(pColData, NULL, 0); } @@ -1772,7 +1786,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); return tColDataPutValue(pColData, NULL, 0); } @@ -1782,7 +1796,7 @@ static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); return tColDataPutValue(pColData, pData, nData); } @@ -1792,7 +1806,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); return tColDataPutValue(pColData, NULL, 0); } @@ -1806,9 +1820,9 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pBitMap, pColData->nVal, 1); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1820,7 +1834,7 @@ static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); return tColDataPutValue(pColData, pData, nData); } @@ -1834,9 +1848,9 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *p if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pBitMap, pColData->nVal, 0); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1848,7 +1862,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); return tColDataPutValue(pColData, NULL, 0); } @@ -1857,7 +1871,7 @@ static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2); return tColDataPutValue(pColData, pData, nData); } @@ -1866,7 +1880,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0); return tColDataPutValue(pColData, NULL, 0); } @@ -1875,7 +1889,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *p code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1); return tColDataPutValue(pColData, NULL, 0); } @@ -2188,6 +2202,304 @@ _exit: return code; } +static int32_t tColDataSwapValue(SColData *pColData, int32_t i, int32_t j) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nData1 = pColData->aOffset[i + 1] - pColData->aOffset[i]; + int32_t nData2 = (j < pColData->nVal - 1) ? pColData->aOffset[j + 1] - pColData->aOffset[j] + : pColData->nData - pColData->aOffset[j]; + uint8_t *pData = taosMemoryMalloc(TMAX(nData1, nData2)); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (nData1 > nData2) { + memcpy(pData, pColData->pData + pColData->aOffset[i], nData1); + memcpy(pColData->pData + pColData->aOffset[i], pColData->pData + pColData->aOffset[j], nData2); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i] + nData1, + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pData, nData1); + } else { + memcpy(pData, pColData->pData + pColData->aOffset[j], nData2); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i], nData1); + memmove(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i] + nData1, + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[i], pData, nData2); + } + for (int32_t k = i + 1; k <= j; ++k) { + pColData->aOffset[k] = pColData->aOffset[k] + nData2 - nData1; + } + + taosMemoryFree(pData); + } else { + uint64_t val; + memcpy(&val, &pColData->pData[TYPE_BYTES[pColData->type] * i], TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * i], &pColData->pData[TYPE_BYTES[pColData->type] * j], + TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * j], &val, TYPE_BYTES[pColData->type]); + } + +_exit: + return code; +} +static void tColDataSwap(SColData *pColData, int32_t i, int32_t j) { + ASSERT(i < j); + ASSERT(j < pColData->nVal); + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + } break; + case HAS_VALUE: { + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT2(pColData->pBitMap, i); + SET_BIT2(pColData->pBitMap, i, GET_BIT2(pColData->pBitMap, j)); + SET_BIT2(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataSort(SColData *aColData, int32_t nColData) { + if (aColData[0].nVal == 0) return; + // TODO +} +static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /* not included */) { + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: { + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_NULL | HAS_NONE): { + if (GET_BIT1(pColData->pBitMap, iStart) == 0) { + for (int32_t i = iStart + 1; i < iEnd; ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 1) { + SET_BIT1(pColData->pBitMap, iStart, 1); + break; + } + } + } + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + if (bv == BIT_FLG_NONE) { + flag |= HAS_NONE; + } else if (bv == BIT_FLG_NULL) { + flag |= HAS_NULL; + } else { + ASSERT(0); + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case HAS_VALUE: { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT1(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { // has a value + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iv != iStart) { + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iv]], + iv < (pColData->nVal - 1) ? pColData->aOffset[iv + 1] - pColData->aOffset[iv] + : pColData->nData - pColData->aOffset[iv]); + } + // TODO + ASSERT(0); + } else { + if (iv != iStart) { + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); + } + memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], + &pColData->pData[TYPE_BYTES[pColData->type] * iEnd], + TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + SET_BIT1(pColData->pBitMap, iStart, 1); + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + uint8_t flag = HAS_VALUE; + for (int32_t i = 0; i < pColData->nVal - (iEnd - iStart - 1); ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 0) { + flag |= HAS_NONE; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } else { // all NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iEnd - 1]], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NULL): { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd - 1, j = iStart; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + if (GET_BIT1(pColData->pBitMap, i)) { + flag |= HAS_VALUE; + } else { + flag |= HAS_NULL; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT2(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { + // TODO + ASSERT(0); + } else { // ALL NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + // TODO + ASSERT(0); + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT2(pColData->pBitMap, j, GET_BIT2(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataMerge(SColData *aColData, int32_t nColData) { + int32_t iStart = 0; + for (;;) { + if (iStart >= aColData[0].nVal - 1) break; + + int32_t iEnd = iStart + 1; + while (iEnd < aColData[0].nVal) { + if (((TSKEY *)aColData[0].pData)[iEnd] != ((TSKEY *)aColData[0].pData)[iStart]) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + for (int32_t i = 0; i < nColData; i++) { + tColDataMergeImpl(&aColData[i], iStart, iEnd); + } + } + + iStart++; + } +} void tColDataSortMerge(SArray *colDataArr) { int32_t nColData = TARRAY_SIZE(colDataArr); SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); @@ -2215,14 +2527,12 @@ void tColDataSortMerge(SArray *colDataArr) { // sort ------- if (doSort) { - ASSERT(0); - // todo + tColDataSort(aColData, nColData); } // merge ------- if (doMerge) { - ASSERT(0); - // todo + tColDataMerge(aColData, nColData); } _exit: diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2e8cb5e5f739110ce75c94bea22aa820862f97d0..f57d59fb41381ecf87ffc56f01e8ee929b7da231 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -55,6 +55,11 @@ int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; +// sync raft +int32_t tsElectInterval = 25 * 1000; +int32_t tsHeartbeatInterval = 1000; +int32_t tsHeartbeatTimeout = 20 * 1000; + // monitor bool tsEnableMonitor = true; int32_t tsMonitorInterval = 30; @@ -74,8 +79,8 @@ char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) -bool tsSmlDataFormat = false; -int32_t tsSmlBatchSize = 10000; +bool tsSmlDataFormat = false; +int32_t tsSmlBatchSize = 10000; // query int32_t tsQueryPolicy = 1; @@ -175,8 +180,6 @@ int32_t tsUptimeInterval = 300; // seconds char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdLdLibPath[512] = ""; -int32_t tsRpcRetryLimit = 100; -int32_t tsRpcRetryInterval = 15; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); @@ -310,8 +313,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; @@ -330,6 +331,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr) != 0) return -1; if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; + if (cfgAddBool(pCfg, "assert", 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, 1) != 0) return -1; if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; @@ -337,7 +339,7 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "AVX", tsAVXEnable, 0) != 0) return -1; if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, 0) != 0) return -1; if (cfgAddBool(pCfg, "FMA", tsFMAEnable, 0) != 0) return -1; - if (cfgAddBool(pCfg, "SIMD-Supported", tsSIMDEnable, 0) != 0) return -1; + if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, 0) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; @@ -404,7 +406,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, 0) != 0) return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); @@ -423,6 +425,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, 0) != 0) return -1; + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, 0) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, 0) != 0) return -1; if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, 0) != 0) return -1; @@ -449,9 +455,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcRetryLimit", tsRpcRetryLimit, 1, 100000, 0) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcRetryInterval", tsRpcRetryInterval, 1, 100000, 0) != 0) return -1; - GRANT_CFG_ADD; return 0; } @@ -666,8 +669,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; - tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32; - tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; return 0; } @@ -686,6 +687,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; taosSetCoreDump(enableCore); + tsAssert = cfgGetItem(pCfg, "assert")->bval; + // todo tsVersion = 30000000; } @@ -737,15 +740,16 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsWalFsyncDataSizeLimit = cfgGetItem(pCfg, "walFsyncDataSizeLimit")->i64; + tsElectInterval = cfgGetItem(pCfg, "syncElectInterval")->i32; + tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; + tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; + tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } - - tsRpcRetryLimit = cfgGetItem(pCfg, "rpcRetryLimit")->i32; - tsRpcRetryInterval = cfgGetItem(pCfg, "rpcRetryInterval")->i32; GRANT_CFG_GET; return 0; } @@ -773,6 +777,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { case 'a': { if (strcasecmp("asyncLog", name) == 0) { tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + } else if (strcasecmp("assert", name) == 0) { + tsAssert = cfgGetItem(pCfg, "assert")->bval; } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8dc4633443e3224254fe72a64e9313f357014c42..95a72843e87ffce615d3a41e6ea666130900976e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -28,6 +28,8 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" +#include "tlog.h" + int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; @@ -551,6 +553,8 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (pReq->ast2Len > 0) { if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1; } + if (tEncodeI64(&encoder, pReq->deleteMark1) < 0) return -1; + if (tEncodeI64(&encoder, pReq->deleteMark2) < 0) return -1; tEndEncode(&encoder); @@ -644,6 +648,9 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1; } + if (tDecodeI64(&decoder, &pReq->deleteMark1) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->deleteMark2) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -822,6 +829,7 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq if (pReq->astLen > 0) { if (tEncodeBinary(&encoder, pReq->ast, pReq->astLen) < 0) return -1; } + if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -870,7 +878,7 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } - + if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -992,7 +1000,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI32(&encoder, vlen) < 0) return -1; for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); - int64_t reserved = 0; + int64_t reserved = 0; if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; if (tEncodeI8(&encoder, pload->syncState) < 0) return -1; if (tEncodeI8(&encoder, pload->syncRestore) < 0) return -1; @@ -5696,6 +5704,25 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } +void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { + if (pReq == NULL) return; + + if (flags & TSDB_MSG_FLG_ENCODE) { + // TODO + } else if (flags & TSDB_MSG_FLG_DECODE) { + if (pReq->comment) { + pReq->comment = NULL; + taosMemoryFree(pReq->comment); + } + + if (pReq->type == TSDB_CHILD_TABLE) { + if (pReq->ctb.tagName) taosArrayDestroy(pReq->ctb.tagName); + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (pReq->ntb.schemaRow.pSchema) taosMemoryFree(pReq->ntb.schemaRow.pSchema); + } + } +} + int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) { int32_t nReq = taosArrayGetSize(pReq->pArray); @@ -6867,11 +6894,16 @@ _exit: } void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { - if (pTbData->pCreateTbReq) { - taosMemoryFree(pTbData->pCreateTbReq); + if (NULL == pTbData) { + return; } if (flag == TSDB_MSG_FLG_ENCODE) { + if (pTbData->pCreateTbReq) { + tdDestroySVCreateTbReq(pTbData->pCreateTbReq); + taosMemoryFree(pTbData->pCreateTbReq); + } + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { int32_t nColData = TARRAY_SIZE(pTbData->aCol); SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); @@ -6890,6 +6922,11 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { taosArrayDestroy(pTbData->aRowP); } } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->pCreateTbReq) { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + taosMemoryFree(pTbData->pCreateTbReq); + } + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { taosArrayDestroy(pTbData->aCol); } else { @@ -6975,7 +7012,7 @@ void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { if (NULL == pRsp) { return; } - + if (flag & TSDB_MSG_FLG_ENCODE) { if (pRsp->aCreateTbRsp) { int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 0d47ef1e7f70c791a32feaa6b4075bd9c625d69f..644b253cc21c243152130a488dc2ebf8ad717d01 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -298,8 +298,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -320,7 +320,7 @@ void buildChildTableName(RandTableName* rName) { taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 52ebd7f879f5b4cc6a675d01de2300b6ed24f412..ca2c0567439f22bcc77e23ae69065f065a275764 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "trow.h" +#include "tlog.h" const uint8_t tdVTypeByte[2][3] = {{ // 2 bits diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index a106a09a69a5a1d7d707c2d40548672c384a2987..ec05ef4c449c3ad3cb917a11feb727030d8ee06d 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -23,6 +23,8 @@ #define _DEFAULT_SOURCE #include "ttime.h" +#include "tlog.h" + /* * mktime64 - Converts date to seconds. * Converts Gregorian date to seconds since 1970-01-01 00:00:00. diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 188677656a512be254815709532611013b805745..a8103351b429d2676c2cbaa37bddb2aa17741241 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -17,6 +17,7 @@ #include "dmMgmt.h" #include "mnode.h" #include "tconfig.h" +#include "tglobal.h" // clang-format off #define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'." @@ -45,9 +46,30 @@ static struct { SArray *pArgs; // SConfigPair } global = {0}; -static void dmStopDnode(int signum, void *info, void *ctx) { dmStop(); } +static void dmSetDebugFlag(int32_t signum, void *sigInfo, void *context) { taosSetAllDebugFlag(143, true); } +static void dmSetAssert(int32_t signum, void *sigInfo, void *context) { tsAssert = 1; } + +static void dmStopDnode(int signum, void *sigInfo, void *context) { + // taosIgnSignal(SIGUSR1); + // taosIgnSignal(SIGUSR2); + taosIgnSignal(SIGTERM); + taosIgnSignal(SIGHUP); + taosIgnSignal(SIGINT); + taosIgnSignal(SIGABRT); + taosIgnSignal(SIGBREAK); + + dInfo("shut down signal is %d", signum); +#ifndef WINDOWS + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); +#endif + + dmStop(); +} static void dmSetSignalHandle() { + taosSetSignal(SIGUSR1, dmSetDebugFlag); + taosSetSignal(SIGUSR2, dmSetAssert); taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); @@ -105,6 +127,19 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { return 0; } +static void dmPrintArgs(int32_t argc, char const *argv[]) { + char path[1024] = {0}; + taosGetCwd(path, sizeof(path)); + + char args[1024] = {0}; + int32_t arglen = snprintf(args, sizeof(args), "%s", argv[0]); + for (int32_t i = 1; i < argc; ++i) { + arglen = arglen + snprintf(args + arglen, sizeof(args) - arglen, " %s", argv[i]); + } + + dInfo("startup path:%s args:%s", path, args); +} + static void dmGenerateGrant() { mndGenerateMachineCode(); } static void dmPrintVersion() { @@ -194,6 +229,8 @@ int mainWindows(int argc, char **argv) { return -1; } + dmPrintArgs(argc, argv); + if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { dError("failed to start since read config error"); taosCloseLog(); @@ -201,7 +238,12 @@ int mainWindows(int argc, char **argv) { return -1; } - taosConvInit(); + if (taosConvInit() != 0) { + dError("failed to init conv"); + taosCloseLog(); + taosCleanupArgs(); + return -1; + } if (global.dumpConfig) { dmDumpCfg(); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index a7ad983b0c960ca500233bf9158e583c16bcbb37..39eeeb32ea24e23acc534a6e4b7ff414fbf91ad8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -103,7 +103,12 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { tSerializeSStatusReq(pHead, contLen, &req); tFreeSStatusReq(&req); - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527}; + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_STATUS, + .info.ahandle = (void *)0x9527, + .info.refId = 0, + .info.noResp = 0}; SRpcMsg rpcRsp = {0}; dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); @@ -150,7 +155,8 @@ static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { SServerStatusRsp statusRsp = {0}; SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); - if (minfo.isMnode && (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { + if (minfo.isMnode && + (minfo.load.syncState == TAOS_SYNC_STATE_ERROR || minfo.load.syncState == TAOS_SYNC_STATE_OFFLINE)) { pStatus->statusCode = TSDB_SRV_STATUS_SERVICE_DEGRADED; snprintf(pStatus->details, sizeof(pStatus->details), "mnode sync state is %s", syncStr(minfo.load.syncState)); return; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 30ef7b9542be6a3365dcd49284d96f4edbb26016..80c040a5e8f84a8d5d7ea887b8a633f945576e17 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -20,7 +20,9 @@ static void *dmStatusThreadFp(void *param) { SDnodeMgmt *pMgmt = param; int64_t lastTime = taosGetTimestampMs(); setThreadName("dnode-status"); - + + const static int16_t TRIM_FREQ = 30; + int32_t trimCount = 0; while (1) { taosMsleep(200); if (pMgmt->pData->dropped || pMgmt->pData->stopped) break; @@ -28,9 +30,13 @@ static void *dmStatusThreadFp(void *param) { int64_t curTime = taosGetTimestampMs(); float interval = (curTime - lastTime) / 1000.0f; if (interval >= tsStatusInterval) { - taosMemoryTrim(0); dmSendStatusReq(pMgmt); lastTime = curTime; + + trimCount = (trimCount + 1) % TRIM_FREQ; + if (trimCount == 0) { + taosMemoryTrim(0); + } } } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 212de0bfb4c298247210536aaa557c1f68561dc4..095857825d3e02753a9be3a257af54815dabc973 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -61,7 +61,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { pMsg->info.rsp = NULL; } - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { mndPostProcessQueryMsg(pMsg); } @@ -159,12 +159,12 @@ int32_t mmPutMsgToQueue(SMnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { } if (pWorker == NULL) return -1; - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) return -1; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); pRpc->pCont = NULL; - dTrace("msg:%p, is created and will put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + dTrace("msg:%p, is created and will put into %s queue, type:%s len:%d", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType), pRpc->contLen); int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); if (code != 0) { dTrace("msg:%p, is freed", pMsg); diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 3e5ad65db759dbff8ab29b9878885019ee428b48..28da0f9c5f5e552416bca638c8ec10e4d9443683 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -58,19 +58,19 @@ int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) return -1; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); pRpc->pCont = NULL; switch (qtype) { case QUERY_QUEUE: - dTrace("msg:%p, is created and will put into qnode-query queue", pMsg); + dTrace("msg:%p, is created and will put into qnode-query queue, len:%d", pMsg, pRpc->contLen); taosWriteQitem(pMgmt->queryWorker.queue, pMsg); return 0; case READ_QUEUE: case FETCH_QUEUE: - dTrace("msg:%p, is created and will put into qnode-fetch queue", pMsg); + dTrace("msg:%p, is created and will put into qnode-fetch queue, len:%d", pMsg, pRpc->contLen); taosWriteQitem(pMgmt->fetchWorker.queue, pMsg); return 0; default: diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index 2d2a121795c72e39fbd858f3ae457b1990e450da..9bd5be520196caea28c13aa9c860c05d28ac7248 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -130,7 +130,7 @@ void smStopWorker(SSnodeMgmt *pMgmt) { } int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) { rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; @@ -139,8 +139,8 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { SSnode *pSnode = pMgmt->pSnode; if (pSnode == NULL) { - dError("snode: msg:%p failed to put into vnode queue since %s, type:%s qtype:%d", pMsg, terrstr(), - TMSG_INFO(pMsg->msgType), qtype); + dError("msg:%p failed to put into snode queue since %s, type:%s qtype:%d len:%d", pMsg, terrstr(), + TMSG_INFO(pMsg->msgType), qtype, pRpc->contLen); taosFreeQitem(pMsg); rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; @@ -161,7 +161,8 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { smPutNodeMsgToWriteQueue(pMgmt, pMsg); break; default: - ASSERT(0); + ASSERTS(0, "msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(), + TMSG_INFO(pMsg->msgType), qtype); } return 0; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index b38dc19361a0bedf9cf1f22b34d3fc074e1172b4..b5c554e0caf83724e7e2bb8b157793765527a844 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -38,6 +38,8 @@ typedef struct SVnodeMgmt { TdThreadRwlock lock; SVnodesStat state; STfs *pTfs; + TdThread thread; + bool stop; } SVnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e15d7ac3dfa122f4204a4cd8301920ba1fba2cd1..bc4677285808e3d8fd76d9f8d65da7e3ab7249f6 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -216,7 +216,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { dDebug("vgId:%d, already exist", req.vgId); tFreeSCreateVnodeReq(&req); vmReleaseVnode(pMgmt, pVnode); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + terrno = TSDB_CODE_VND_ALREADY_EXIST; code = terrno; return 0; } @@ -307,7 +307,7 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dError("vgId:%d, failed to alter replica since %s", vgId, terrstr()); - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } @@ -369,7 +369,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + terrno = TSDB_CODE_VND_NOT_EXIST; return -1; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 07ebd72379e29d26d2ebbe830f0df9c8458aa969..313a88fc5c60f3ac2ae84b1ca6cf811033f5779c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -334,6 +334,62 @@ static void vmCleanup(SVnodeMgmt *pMgmt) { taosMemoryFree(pMgmt); } +static void vmCheckSyncTimeout(SVnodeMgmt *pMgmt) { + int32_t numOfVnodes = 0; + SVnodeObj **ppVnodes = vmGetVnodeListFromHash(pMgmt, &numOfVnodes); + + for (int32_t i = 0; i < numOfVnodes; ++i) { + SVnodeObj *pVnode = ppVnodes[i]; + vnodeSyncCheckTimeout(pVnode->pImpl); + vmReleaseVnode(pMgmt, pVnode); + } + + if (ppVnodes != NULL) { + taosMemoryFree(ppVnodes); + } +} + +static void *vmThreadFp(void *param) { + SVnodeMgmt *pMgmt = param; + int64_t lastTime = 0; + setThreadName("vnode-timer"); + + while (1) { + lastTime++; + taosMsleep(100); + if (pMgmt->stop) break; + if (lastTime % 10 != 0) continue; + + int64_t sec = lastTime / 10; + if (sec % (VNODE_TIMEOUT_SEC / 2) == 0) { + vmCheckSyncTimeout(pMgmt); + } + } + + return NULL; +} + +static int32_t vmInitTimer(SVnodeMgmt *pMgmt) { + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + if (taosThreadCreate(&pMgmt->thread, &thAttr, vmThreadFp, pMgmt) != 0) { + dError("failed to create vnode timer thread since %s", strerror(errno)); + return -1; + } + + taosThreadAttrDestroy(&thAttr); + return 0; +} + +static void vmCleanupTimer(SVnodeMgmt *pMgmt) { + pMgmt->stop = true; + if (taosCheckPthreadValid(pMgmt->thread)) { + taosThreadJoin(pMgmt->thread, NULL); + taosThreadClear(&pMgmt->thread); + } +} + static int32_t vmInit(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { int32_t code = -1; @@ -510,12 +566,10 @@ static int32_t vmStartVnodes(SVnodeMgmt *pMgmt) { taosMemoryFree(ppVnodes); } - return 0; + return vmInitTimer(pMgmt); } -static void vmStop(SVnodeMgmt *pMgmt) { - // process inside the vnode -} +static void vmStop(SVnodeMgmt *pMgmt) { vmCleanupTimer(pMgmt); } SMgmtFunc vmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 08ea880b97fa6add105839f2e28f19bcc064e816..7e3915f3d1dcdfc713746aad6af1bd609231d001 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -233,7 +233,7 @@ int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { - SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) { rpcFreeCont(pRpc->pCont); pRpc->pCont = NULL; @@ -241,7 +241,7 @@ int32_t vmPutRpcMsgToQueue(SVnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { } SMsgHead *pHead = pRpc->pCont; - dTrace("vgId:%d, msg:%p is created, type:%s", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType)); + dTrace("vgId:%d, msg:%p is created, type:%s len:%d", pHead->vgId, pMsg, TMSG_INFO(pRpc->msgType), pRpc->contLen); pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 421d723202b0f7fb9824cab2fa0a568f4cf38e69..e3bda5a3f01355f33bdf758e0a14824162c50a2b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -152,7 +152,19 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); if (pWrapper != NULL) { dmReleaseWrapper(pWrapper); - terrno = TSDB_CODE_NODE_ALREADY_DEPLOYED; + switch (ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_ALREADY_DEPLOYED; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_ALREADY_DEPLOYED; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_ALREADY_DEPLOYED; + break; + default: + terrno = TSDB_CODE_APP_ERROR; + } dError("failed to create node since %s", terrstr()); return -1; } @@ -191,7 +203,20 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); if (pWrapper == NULL) { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + switch (ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_NOT_DEPLOYED; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_NOT_DEPLOYED; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_NOT_DEPLOYED; + break; + default: + terrno = TSDB_CODE_APP_ERROR; + } + dError("failed to drop node since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index 2c9020b668550a914cdd7e235ad2c5242af36057..b6cce249ea935a55e0515a84955702e5e463ca4d 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -189,7 +189,6 @@ SMgmtWrapper *dmAcquireWrapper(SDnode *pDnode, EDndNodeType ntype) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); // dTrace("node:%s, is acquired, ref:%d", pWrapper->name, refCount); } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; pRetWrapper = NULL; } taosThreadRwlockUnlock(&pWrapper->lock); @@ -205,7 +204,23 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper) { int32_t refCount = atomic_add_fetch_32(&pWrapper->refCount, 1); // dTrace("node:%s, is marked, ref:%d", pWrapper->name, refCount); } else { - terrno = TSDB_CODE_NODE_NOT_DEPLOYED; + switch (pWrapper->ntype) { + case MNODE: + terrno = TSDB_CODE_MNODE_NOT_FOUND; + break; + case QNODE: + terrno = TSDB_CODE_QNODE_NOT_FOUND; + break; + case SNODE: + terrno = TSDB_CODE_SNODE_NOT_FOUND; + break; + case VNODE: + terrno = TSDB_CODE_VND_STOPPED; + break; + default: + terrno = TSDB_CODE_APP_IS_STOPPING; + break; + } code = -1; } taosThreadRwlockUnlock(&pWrapper->lock); diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 6893e486bb29837f599b4f2226efbc25b61fdb85..981797834ae268e2eeab923a61fa98837a1d4de0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -149,10 +149,13 @@ int32_t dmRunDnode(SDnode *pDnode) { return 0; } - if (count == 0) osUpdate(); - - count %= 10; - count++; + if (count == 10) { + osUpdate(); + count = 0; + } else { + count++; + } + taosMsleep(100); } } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index d037cc33752fd1255e1e6dba967e9214a3cea664..5e1dcc6353734947706e91697e6882c695e8a508 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -33,23 +33,6 @@ static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { } } -static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { - pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info, .msgType = pMsg->msgType}; - int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); - - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet); - rsp.contLen = contLen; - } - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} - int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; @@ -65,6 +48,11 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { return (*msgFp)(pWrapper->pMgmt, pMsg); } +static bool dmFailFastFp(tmsg_t msgType) { + // add more msg type later + return msgType == TDMT_SYNC_HEARTBEAT || msgType == TDMT_SYNC_APPEND_ENTRIES; +} + static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; @@ -153,11 +141,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { } pRpc->info.wrapper = pWrapper; - pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); + pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen); if (pMsg == NULL) goto _OVER; memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - dGTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle); + dGTrace("msg:%p, is created, type:%s handle:%p len:%d", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle, + pRpc->contLen); code = dmProcessNodeMsg(pWrapper, pMsg); @@ -172,8 +161,7 @@ _OVER: if (IsReq(pRpc)) { SRpcMsg rsp = {.code = code, .info = pRpc->info}; - if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && - pRpc->msgType < TDMT_VND_MSG) { + if (code == TSDB_CODE_MNODE_NOT_FOUND) { dmBuildMnodeRedirectRsp(pDnode, &rsp); } @@ -226,7 +214,11 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { if (pDnode->status != DND_STAT_RUNNING && pMsg->msgType < TDMT_SYNC_MSG) { rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; - terrno = TSDB_CODE_NODE_OFFLINE; + if (pDnode->status == DND_STAT_INIT) { + terrno = TSDB_CODE_APP_IS_STARTING; + } else { + terrno = TSDB_CODE_APP_IS_STOPPING; + } dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { @@ -240,9 +232,9 @@ static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { rpcRegisterBrokenLin static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { rpcReleaseHandle(pHandle, type); } static bool rpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK || - code == TSDB_CODE_VND_STOPPED) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_MNODE_NOT_FOUND || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_APP_IS_STARTING || code == TSDB_CODE_APP_IS_STOPPING) { if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { return false; @@ -267,13 +259,15 @@ int32_t dmInitClient(SDnode *pDnode) { rpcInit.rfp = rpcRfp; rpcInit.compressSize = tsCompressMsgSize; - rpcInit.retryLimit = tsRpcRetryLimit; - rpcInit.retryInterval = tsRpcRetryInterval; rpcInit.retryMinInterval = tsRedirectPeriod; rpcInit.retryStepFactor = tsRedirectFactor; rpcInit.retryMaxInterval = tsRedirectMaxPeriod; rpcInit.retryMaxTimouet = tsMaxRetryWaitTime; + rpcInit.failFastInterval = 1000; // interval threshold(ms) + rpcInit.failFastThreshold = 3; // failed threshold + rpcInit.ffp = dmFailFastFp; + pTrans->clientRpc = rpcOpen(&rpcInit); if (pTrans->clientRpc == NULL) { dError("failed to init dnode rpc client"); @@ -306,6 +300,7 @@ int32_t dmInitServer(SDnode *pDnode) { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.parent = pDnode; + rpcInit.compressSize = tsCompressMsgSize; pTrans->serverRpc = rpcOpen(&rpcInit); if (pTrans->serverRpc == NULL) { @@ -331,7 +326,6 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) { .clientRpc = pDnode->trans.clientRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, - .sendRedirectRspFp = dmSendRedirectRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, .releaseHandleFp = dmReleaseHandle, .reportStartupFp = dmReportStartup, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 8719e988e7acd44548708a83f9e846d981170712..2124b387ec926a3add0c5efe3a51f43bdf9ae47c 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -39,7 +39,7 @@ #include "sync.h" #include "wal.h" -#include "libs/function/function.h" +#include "libs/function/tudf.h" #ifdef __cplusplus extern "C" { #endif diff --git a/source/dnode/mgmt/test/mnode/dmnode.cpp b/source/dnode/mgmt/test/mnode/dmnode.cpp index 7b7eb97216d9f443296b56393a273b5ccb01d9e9..28cb376b3789c284daed7bc7b745b00d802c6a29 100644 --- a/source/dnode/mgmt/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/test/mnode/dmnode.cpp @@ -40,7 +40,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } { @@ -57,7 +57,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } { @@ -77,7 +77,7 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_ALREADY_DEPLOYED); } } @@ -171,7 +171,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED); } { @@ -188,7 +188,7 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_MNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/qnode/dqnode.cpp b/source/dnode/mgmt/test/qnode/dqnode.cpp index a2c6a2c28c81b7ba4ae76bd8e843696d879bf692..3beb57c516cda1c1398514a0213ee35ad9b0d500 100644 --- a/source/dnode/mgmt/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/test/qnode/dqnode.cpp @@ -62,7 +62,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_ALREADY_DEPLOYED); } } @@ -120,7 +120,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); } test.Restart(); @@ -135,7 +135,7 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_QNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/snode/dsnode.cpp b/source/dnode/mgmt/test/snode/dsnode.cpp index e3ad65d8316fdb42e44d632e3fcfb20483377e53..30d6a348139731ff9e255b21c2bb64a38b0c5500 100644 --- a/source/dnode/mgmt/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/test/snode/dsnode.cpp @@ -62,7 +62,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); } test.Restart(); @@ -77,7 +77,7 @@ TEST_F(DndTestSnode, 01_Create_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_ALREADY_DEPLOYED); } } @@ -120,7 +120,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); } test.Restart(); @@ -135,7 +135,7 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_SNODE_NOT_DEPLOYED); } { diff --git a/source/dnode/mgmt/test/vnode/vnode.cpp b/source/dnode/mgmt/test/vnode/vnode.cpp index 8a8e3322891c60d067941cd886b8b6498c8346b1..24e7affd887fd6b5f18320b205f9c9b0f65a3c28 100644 --- a/source/dnode/mgmt/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/test/vnode/vnode.cpp @@ -63,7 +63,7 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { ASSERT_EQ(pRsp->code, 0); test.Restart(); } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_ALREADY_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_VND_ALREADY_EXIST); } } } @@ -285,7 +285,7 @@ TEST_F(DndTestVnode, 06_Drop_Vnode) { ASSERT_EQ(pRsp->code, 0); test.Restart(); } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_NODE_NOT_DEPLOYED); + ASSERT_EQ(pRsp->code, TSDB_CODE_VND_NOT_EXIST); } } } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9961828747f290cf501ceb2c51ee744b4a31349c..3c42f7b832ef5d3c106c1886a2609677b17767bf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -473,6 +473,7 @@ void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; + char createUser[TSDB_USER_LEN]; int64_t createTime; int64_t updateTime; int64_t uid; @@ -644,6 +645,7 @@ typedef struct { // 3.0.20 int64_t checkpointFreq; // ms int64_t currentTick; // do not serialize + int64_t deleteMark; } SStreamObj; int32_t tEncodeSStreamObj(SEncoder* pEncoder, const SStreamObj* pObj); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index a0f3c98f8340f44e1ca940008a69835eb0f4cbcf..785ecc2bf502ca5f2a1e1203f1dc5fbe29b60a21 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -84,14 +84,16 @@ typedef struct { } STelemMgmt; typedef struct { - tsem_t syncSem; - int64_t sync; - int32_t errCode; - int32_t transId; - SRWLatch lock; - int8_t selfIndex; - int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + tsem_t syncSem; + int64_t sync; + int32_t errCode; + int32_t transId; + int32_t transSec; + int64_t transSeq; + TdThreadMutex lock; + int8_t selfIndex; + int8_t numOfReplicas; + SReplica replicas[TSDB_MAX_REPLICA]; } SSyncMgmt; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndScheduler.h b/source/dnode/mnode/impl/inc/mndScheduler.h index db81d8843e5415b4bf11fd33a013fb700f5914bc..23085c53eed7ef6234da38f146f962ea96d9fdde 100644 --- a/source/dnode/mnode/impl/inc/mndScheduler.h +++ b/source/dnode/mnode/impl/inc/mndScheduler.h @@ -28,7 +28,7 @@ void mndCleanupScheduler(SMnode* pMnode); int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub); int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, - int64_t watermark); + int64_t watermark, int64_t deleteMark); int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream); diff --git a/source/dnode/mnode/impl/inc/mndSync.h b/source/dnode/mnode/impl/inc/mndSync.h index 993efbcd084443fccd8ef9776b942e7af671bf60..1f9a698a8ad8b5a0f67ad18168938d3e1de0f31d 100644 --- a/source/dnode/mnode/impl/inc/mndSync.h +++ b/source/dnode/mnode/impl/inc/mndSync.h @@ -26,6 +26,7 @@ int32_t mndInitSync(SMnode *pMnode); void mndCleanupSync(SMnode *pMnode); bool mndIsLeader(SMnode *pMnode); int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId); +void mndSyncCheckTimeout(SMnode *pMnode); void mndSyncStart(SMnode *pMnode); void mndSyncStop(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 2372fa30e5bfc1e27b32ff10672929a328029865..07066d2251235d325aa9f64f2751c8248e8c7e54 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -75,6 +75,7 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, voi void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname); void mndTransSetSerial(STrans *pTrans); void mndTransSetOper(STrans *pTrans, EOperType oper); +int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransProcessRsp(SRpcMsg *pRsp); diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 970d1db7dbc2ff9a92f201aabb940fdeea1f22e4..cf7deba397556f8df3550067057a03f6ca374a2a 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -31,6 +31,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); SHashObj *mndDupDbHash(SHashObj *pOld); +SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index ce2ec7b23aaaff451b44f470125abf8e8fb87331..148e21b507cdfe49ce6bab50fe4d9d233d9172f2 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -220,7 +220,7 @@ static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -230,7 +230,7 @@ static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } @@ -240,7 +240,7 @@ static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { return -1; } - terrno = TSDB_CODE_MSG_NOT_PROCESSED; + terrno = TSDB_CODE_OPS_NOT_SUPPORT; mError("failed to process create acct request since %s", terrstr()); return -1; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 76bff9a70ff9ba09d852ada451d83cbdc397c0ca..0a00980c14422e441e97411bed35c0c383bfecdd 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -554,7 +554,12 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } - if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { + mndReleaseTopic(pMnode, pTopic); + goto SUBSCRIBE_OVER; + } + + if (mndCheckTopicPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_SUBSCRIBE, pTopic) != 0) { goto SUBSCRIBE_OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5fa86dffe289ad7410cc9cbc7b58edf800bba966..0715556da2966b2f09d279ef2b10bcea0b6185d9 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -776,6 +776,8 @@ static int32_t mndAlterDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pOld, SDbObj *p int32_t code = -1; mndTransSetDbName(pTrans, pOld->name, NULL); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) return -1; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; @@ -825,16 +827,24 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndAlterDb(pMnode, pReq, pDb, &dbObj); - if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + + if (dbObj.cfg.replications != pDb->cfg.replications) { + // return quickly, operation executed asynchronously + mInfo("db:%s, alter db replica from %d to %d", pDb->name, pDb->cfg.replications, dbObj.cfg.replications); + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + if (terrno != 0) code = terrno; mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); taosArrayDestroy(dbObj.cfg.pRetensions); + terrno = code; return code; } @@ -1177,7 +1187,8 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || pReq->stateTs < pDb->stateTs) { + if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable || + pReq->stateTs < pDb->stateTs) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); } @@ -1292,21 +1303,22 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, SUseDbRsp usedbRsp = {0}; - if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { + if ((0 == strcasecmp(pDbVgVersion->dbFName, TSDB_INFORMATION_SCHEMA_DB) || + (0 == strcasecmp(pDbVgVersion->dbFName, TSDB_PERFORMANCE_SCHEMA_DB)))) { memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode); if (pDbVgVersion->vgVersion < vgVersion) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); - + mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; } else { usedbRsp.vgVersion = pDbVgVersion->vgVersion; } usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - + taosArrayPush(batchUseRsp.pArray, &usedbRsp); - + continue; } @@ -1322,7 +1334,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); - if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && + if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable && pDbVgVersion->stateTs == pDb->stateTs) { mTrace("db:%s, valid dbinfo, vgVersion:%d stateTs:%" PRId64 " numOfTables:%d, not changed vgVersion:%d stateTs:%" PRId64 " numOfTables:%d", diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 250a302cc44b5b8a923db6e920d4dc11700e4f2e..5772864ba82b27390a294e5a88465c21cb4f4b0e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -383,9 +383,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { pGid->syncCanRead != pVload->syncCanRead) { mInfo( "vgId:%d, state changed by status msg, old state:%s restored:%d canRead:%d new state:%s restored:%d " - "canRead:%d", + "canRead:%d, dnode:%d", pVgroup->vgId, syncStr(pGid->syncState), pGid->syncRestore, pGid->syncCanRead, - syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead); + syncStr(pVload->syncState), pVload->syncRestore, pVload->syncCanRead, pDnode->id); pGid->syncState = pVload->syncState; pGid->syncRestore = pVload->syncRestore; pGid->syncCanRead = pVload->syncCanRead; @@ -672,6 +672,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { snprintf(ep, TSDB_EP_LEN, "%s:%d", createReq.fqdn, createReq.port); pDnode = mndAcquireDnodeByEp(pMnode, ep); if (pDnode != NULL) { + terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST; goto _OVER; } @@ -787,7 +788,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { int32_t numOfVnodes = mndGetVnodesNum(pMnode, pDnode->id); if ((numOfVnodes > 0 || pMObj != NULL || pSObj != NULL || pQObj != NULL) && !dropReq.force) { if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; mError("dnode:%d, failed to drop since %s, vnodes:%d mnode:%d qnode:%d snode:%d", pDnode->id, terrstr(), numOfVnodes, pMObj != NULL, pQObj != NULL, pSObj != NULL); goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 09172115f8502e392c1d37ae1d256761afb02126..82294ac7bf1d499d93bff63b6dbdea07577ba730 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -71,7 +71,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) { int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, bool sysinfo, STableMetaRsp *pRsp) { if (NULL == pMnode->infosMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -97,7 +97,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { if (NULL == pMnode->infosMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index d46ae7aaa985ddc1a51c12eea72bb411477bb7f5..854535c82f72bf5ca4c3340355980ecad07ff7df 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -45,7 +45,7 @@ static inline int32_t mndAcquireRpc(SMnode *pMnode) { int32_t code = 0; taosThreadRwlockRdlock(&pMnode->lock); if (pMnode->stopped) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_IS_STOPPING; code = -1; } else if (!mndIsLeader(pMnode)) { code = -1; @@ -101,6 +101,7 @@ static void *mndBuildCheckpointTickMsg(int32_t *pContLen, int64_t sec) { } static void mndPullupTrans(SMnode *pMnode) { + mTrace("pullup trans msg"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -110,6 +111,7 @@ static void mndPullupTrans(SMnode *pMnode) { } static void mndPullupTtl(SMnode *pMnode) { + mTrace("pullup ttl"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; @@ -117,6 +119,7 @@ static void mndPullupTtl(SMnode *pMnode) { } static void mndCalMqRebalance(SMnode *pMnode) { + mTrace("calc mq rebalance"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -143,6 +146,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { } static void mndPullupTelem(SMnode *pMnode) { + mTrace("pullup telem msg"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -152,6 +156,7 @@ static void mndPullupTelem(SMnode *pMnode) { } static void mndPullupGrant(SMnode *pMnode) { + mTrace("pullup grant msg"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -162,6 +167,7 @@ static void mndPullupGrant(SMnode *pMnode) { } static void mndIncreaseUpTime(SMnode *pMnode) { + mTrace("increate uptime"); int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { @@ -213,6 +219,9 @@ static void mndSetVgroupOffline(SMnode *pMnode, int32_t dnodeId, int64_t curMs) } static void mndCheckDnodeOffline(SMnode *pMnode) { + mTrace("check dnode offline"); + if (mndAcquireRpc(pMnode) != 0) return; + SSdb *pSdb = pMnode->pSdb; int64_t curMs = taosGetTimestampMs(); @@ -230,6 +239,8 @@ static void mndCheckDnodeOffline(SMnode *pMnode) { sdbRelease(pSdb, pDnode); } + + mndReleaseRpc(pMnode); } static void *mndThreadFp(void *param) { @@ -277,6 +288,10 @@ static void *mndThreadFp(void *param) { if (sec % (tsStatusInterval * 5) == 0) { mndCheckDnodeOffline(pMnode); } + + if (sec % (MNODE_TIMEOUT_SEC / 2) == 0) { + mndSyncCheckTimeout(pMnode); + } } return NULL; @@ -599,11 +614,45 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_SCH_MERGE_FETCH || pMsg->msgType == TDMT_SCH_DROP_TASK) { return 0; } - if (mndAcquireRpc(pMsg->info.node) == 0) return 0; - SMnode *pMnode = pMsg->info.node; + SMnode *pMnode = pMsg->info.node; + taosThreadRwlockRdlock(&pMnode->lock); + if (pMnode->stopped) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_APP_IS_STOPPING; + return -1; + } + + terrno = 0; SSyncState state = syncGetState(pMnode->syncMgmt.sync); + if (terrno != 0) { + taosThreadRwlockUnlock(&pMnode->lock); + return -1; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_NOT_LEADER; + goto _OVER; + } + if (!state.restored || !pMnode->restored) { + taosThreadRwlockUnlock(&pMnode->lock); + terrno = TSDB_CODE_SYN_RESTORING; + goto _OVER; + } + +#if 1 + atomic_add_fetch_32(&pMnode->rpcRef, 1); +#else + int32_t ref = atomic_add_fetch_32(&pMnode->rpcRef, 1); + mTrace("mnode rpc is acquired, ref:%d", ref); +#endif + + taosThreadRwlockUnlock(&pMnode->lock); + return 0; + +_OVER: if (pMsg->msgType == TDMT_MND_TMQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER || pMsg->msgType == TDMT_MND_UPTIME_TIMER) { @@ -614,31 +663,28 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; SEpSet epSet = {0}; + int32_t tmpCode = terrno; mndGetMnodeEpSet(pMnode, &epSet); + terrno = tmpCode; - mDebug( + mGDebug( "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " "role:%s, redirect numOfEps:%d inUse:%d", pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, syncStr(state.restored), epSet.numOfEps, epSet.inUse); - if (epSet.numOfEps > 0) { - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); - } + if (epSet.numOfEps <= 0) return -1; - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->info.rsp = rpcMallocCont(contLen); - if (pMsg->info.rsp != NULL) { - tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); - pMsg->info.hasEpSet = 1; - pMsg->info.rspLen = contLen; - terrno = TSDB_CODE_RPC_REDIRECT; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } - } else { - terrno = TSDB_CODE_APP_NOT_READY; + for (int32_t i = 0; i < epSet.numOfEps; ++i) { + mDebug("mnode index:%d, ep:%s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + } + + int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->info.rsp = rpcMallocCont(contLen); + if (pMsg->info.rsp != NULL) { + tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.hasEpSet = 1; + pMsg->info.rspLen = contLen; } return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 2620100b6ca46cdebaa64cc892bcbdf697c5f458..9c847c11380f703a92b56f713c71b3a9c4e5f23a 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -292,7 +292,7 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p .pCont = pReq, .contLen = contLen, .msgType = TDMT_DND_CREATE_MNODE, - .acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, }; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -333,7 +333,7 @@ static int32_t mndBuildDropMnodeRedoAction(STrans *pTrans, SDDropMnodeReq *pDrop .pCont = pReq, .contLen = contLen, .msgType = TDMT_DND_DROP_MNODE, - .acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED, + .acceptableCode = TSDB_CODE_MNODE_NOT_DEPLOYED, }; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -440,7 +440,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } if (!mndIsDnodeOnline(pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; goto _OVER; } @@ -490,7 +490,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (totalMnodes == 2) { if (force) { mError("cant't force drop dnode, since a mnode on it and replica is 2"); - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; return -1; } mInfo("vgId:1, has %d mnodes, exec redo log first", totalMnodes); @@ -574,7 +574,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } if (!mndIsDnodeOnline(pObj->pDnode, taosGetTimestampMs())) { - terrno = TSDB_CODE_NODE_OFFLINE; + terrno = TSDB_CODE_DNODE_OFFLINE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index e874c1a53e6d5e0aaffe4891fb25dca3b6a11d07..cf463304a1059a1cb1689bd02cc9d6fc21417ad5 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -68,7 +68,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { if (NULL == pMnode->perfsMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } @@ -94,7 +94,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { if (NULL == pMnode->perfsMeta) { - terrno = TSDB_CODE_APP_NOT_READY; + terrno = TSDB_CODE_APP_ERROR; return -1; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d42b66455f2be95c160e9d39d8ea84a3cc10ac0a..ffc357b2e801f0bb51ad59dee2fc57a7cfad17bf 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -542,7 +542,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } default: mError("invalid kv key:%d", kv->key); - hbRsp.status = TSDB_CODE_MND_APP_ERROR; + hbRsp.status = TSDB_CODE_APP_ERROR; break; } @@ -769,7 +769,7 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p } int32_t numOfQueries = taosArrayGetSize(pConn->pQueries); - for (int32_t i = 0; i < numOfQueries; ++i) { + for (int32_t i = 0; i < numOfQueries && numOfRows < rows; ++i) { SQueryDesc *pQuery = taosArrayGet(pConn->pQueries, i); cols = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 26e7f72cf72eb792e07aa1dc6c49b80f2c612497..28a5dee2dbd6a5590e057318efcd76fdd2a7f6a9 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -205,7 +205,7 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -232,7 +232,7 @@ static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -345,7 +345,7 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_QNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index f71bd1c626e60fc7e6d8bd7aec2a02ddc82bf869..af1a29def08eae42072fffc889cd494a95dea239 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -42,7 +42,7 @@ static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { } int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType, - int64_t watermark) { + int64_t watermark, int64_t deleteMark) { SNode* pAst = NULL; SQueryPlan* pPlan = NULL; terrno = TSDB_CODE_SUCCESS; @@ -64,6 +64,7 @@ int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64 .rSmaQuery = true, .triggerType = triggerType, .watermark = watermark, + .deleteMark = deleteMark, }; if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 20c2ebb0a4789ff74832c3bc029f6dd6f4241612..6a7e2aaa51b2f7603c186ac75529e231561f442a 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -108,6 +108,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_APPS; } else if (strncasecmp(name, TSDB_INS_TABLE_STREAM_TASKS, len) == 0) { type = TSDB_MGMT_TABLE_STREAM_TASKS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) { + type = TSDB_MGMT_TABLE_PRIVILEGES; } else { // ASSERT(0); } @@ -196,9 +198,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } if (retrieveReq.showId == 0) { - STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); + STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { - pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); + pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 7be2d48c703c5c80df4fea6a7d8c5c411dc429d4..e6a6e6ae5b326cf956052ac470b9c8db6a3a0ab4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -463,7 +463,7 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -534,6 +534,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.sql = strdup(pCreate->sql); streamObj.smaId = smaObj.uid; streamObj.watermark = pCreate->watermark; + streamObj.deleteMark = pCreate->deleteMark; streamObj.fillHistory = STREAM_FILL_HISTORY_ON; streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; streamObj.triggerParam = pCreate->maxDelay; @@ -574,6 +575,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea .streamQuery = true, .triggerType = streamObj.trigger, .watermark = streamObj.watermark, + .deleteMark = streamObj.deleteMark, }; if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { @@ -780,7 +782,7 @@ static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SD action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a31801f3767b60d0a72243c3c0c8957d0ce98818..e6a253bcc08c35d28bf7e74bf5d2804573e134db 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -210,7 +210,7 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -237,7 +237,7 @@ static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, S action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -352,7 +352,7 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_SNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 2ed4fde50963418af86be25a32f10f9e609c00e9..c611cfb6e1d8adf8c5ae1f0c6f9bb3d2071bb472 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -450,13 +450,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.rsmaParam.watermark[1] = pStb->watermark[1]; if (pStb->ast1Len > 0) { if (mndConvertRsmaTask(&req.rsmaParam.qmsg[0], &req.rsmaParam.qmsgLen[0], pStb->pAst1, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0]) < 0) { + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[0], + req.rsmaParam.deleteMark[0]) < 0) { goto _err; } } if (pStb->ast2Len > 0) { if (mndConvertRsmaTask(&req.rsmaParam.qmsg[1], &req.rsmaParam.qmsgLen[1], pStb->pAst2, pStb->uid, - STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1]) < 0) { + STREAM_TRIGGER_WINDOW_CLOSE, req.rsmaParam.watermark[1], + req.rsmaParam.deleteMark[1]) < 0) { goto _err; } } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7ee688d220931e3a0bf70e93b99a000e0852b5e1..05c05943397ec42daa1d6e35baab6c19b7d2f940 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -164,7 +164,8 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { STREAM_DECODE_OVER: taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { - mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, terrstr()); + mError("stream:%s, failed to decode from raw:%p since %s", pStream == NULL ? "null" : pStream->name, pRaw, + terrstr()); taosMemoryFreeClear(pRow); return NULL; } @@ -624,6 +625,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + pDb = mndAcquireDb(pMnode, streamObj.sourceDb); + if (pDb->cfg.replications != 1) { + mError("stream source db must have only 1 replica, but %s has %d", pDb->name, pDb->cfg.replications); + terrno = TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB; + mndReleaseDb(pMnode, pDb); + pDb = NULL; + goto _OVER; + } + mndReleaseDb(pMnode, pDb); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "create-stream"); if (pTrans == NULL) { mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr()); @@ -680,7 +691,6 @@ _OVER: } mndReleaseStream(pMnode, pStream); - mndReleaseDb(pMnode, pDb); tFreeSCMCreateStreamReq(&createStreamReq); tFreeStreamObj(&streamObj); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 10cd6416b44d1e47528ed0911e9707b546599f36..c96faddc4c69c0360ee55c2bae78741ef4c025d8 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -72,41 +72,44 @@ static int32_t mndSyncSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { return code; } -void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +int32_t mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SMnode *pMnode = pFsm->data; SSyncMgmt *pMgmt = &pMnode->syncMgmt; SSdbRaw *pRaw = pMsg->pCont; int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); - pMgmt->errCode = pMeta->code; mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 - " role:%s raw:%p", + " role:%s raw:%p sec:%d seq:%" PRId64, transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), - pRaw); + pRaw, pMgmt->transSec, pMgmt->transSeq); - if (pMgmt->errCode == 0) { + if (pMeta->code == 0) { sdbWriteWithoutFree(pMnode->pSdb, pRaw); sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); } - taosWLockLatch(&pMgmt->lock); + taosThreadMutexLock(&pMgmt->lock); + pMgmt->errCode = pMeta->code; + if (transId <= 0) { - taosWUnLockLatch(&pMgmt->lock); - mError("trans:%d, invalid commit msg", transId); + taosThreadMutexUnlock(&pMgmt->lock); + mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); } else if (transId == pMgmt->transId) { if (pMgmt->errCode != 0) { mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); } else { - mInfo("trans:%d, is proposed and post sem", transId); + mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq); } pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; tsem_post(&pMgmt->syncSem); - taosWUnLockLatch(&pMgmt->lock); + taosThreadMutexUnlock(&pMgmt->lock); } else { - taosWUnLockLatch(&pMgmt->lock); + taosThreadMutexUnlock(&pMgmt->lock); STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { - mInfo("trans:%d, execute in mnode which not leader", transId); + mInfo("trans:%d, execute in mnode which not leader or sync timeout", transId); mndTransExecute(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); // sdbWriteFile(pMnode->pSdb, SDB_WRITE_DELTA); @@ -114,12 +117,21 @@ void mndProcessWriteMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *p mError("trans:%d, not found while execute in mnode since %s", transId, terrstr()); } } + + return 0; } -void mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - mndProcessWriteMsg(pFsm, pMsg, pMeta); +int32_t mndSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + int32_t code = 0; + if (!syncUtilUserCommit(pMsg->msgType)) { + goto _out; + } + code = mndProcessWriteMsg(pFsm, pMsg, pMeta); + +_out: rpcFreeCont(pMsg->pCont); pMsg->pCont = NULL; + return code; } int32_t mndSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { @@ -140,9 +152,13 @@ void mndRestoreFinish(const SSyncFSM *pFsm) { SMnode *pMnode = pFsm->data; if (!pMnode->deploy) { - mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); - mndTransPullup(pMnode); - mndSetRestored(pMnode, true); + if (!pMnode->restored) { + mInfo("vgId:1, sync restore finished, and will handle outstanding transactions"); + mndTransPullup(pMnode); + mndSetRestored(pMnode, true); + } else { + mInfo("vgId:1, sync restore finished, repeat call"); + } } else { mInfo("vgId:1, sync restore finished"); } @@ -185,18 +201,20 @@ int32_t mndSnapshotDoWrite(const SSyncFSM *pFsm, void *pWriter, void *pBuf, int3 } static void mndBecomeFollower(const SSyncFSM *pFsm) { - SMnode *pMnode = pFsm->data; + SMnode *pMnode = pFsm->data; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; mInfo("vgId:1, become follower"); - taosWLockLatch(&pMnode->syncMgmt.lock); - if (pMnode->syncMgmt.transId != 0) { - mInfo("vgId:1, become follower and post sem, trans:%d, failed to propose since not leader", - pMnode->syncMgmt.transId); - pMnode->syncMgmt.transId = 0; - pMnode->syncMgmt.errCode = TSDB_CODE_SYN_NOT_LEADER; - tsem_post(&pMnode->syncMgmt.syncSem); + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + mInfo("vgId:1, become follower and post sem, trans:%d, failed to propose since not leader", pMgmt->transId); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; + tsem_post(&pMgmt->syncSem); } - taosWUnLockLatch(&pMnode->syncMgmt.lock); + taosThreadMutexUnlock(&pMgmt->lock); } static void mndBecomeLeader(const SSyncFSM *pFsm) { @@ -252,8 +270,10 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { int32_t mndInitSync(SMnode *pMnode) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - taosInitRWLatch(&pMgmt->lock); + taosThreadMutexInit(&pMgmt->lock, NULL); pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; SSyncInfo syncInfo = { .snapshotStrategy = SYNC_STRATEGY_STANDARD_SNAPSHOT, @@ -300,12 +320,38 @@ void mndCleanupSync(SMnode *pMnode) { mInfo("mnode-sync is stopped, id:%" PRId64, pMgmt->sync); tsem_destroy(&pMgmt->syncSem); + taosThreadMutexDestroy(&pMgmt->lock); memset(pMgmt, 0, sizeof(SSyncMgmt)); } +void mndSyncCheckTimeout(SMnode *pMnode) { + mTrace("check sync timeout"); + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + int32_t curSec = taosGetTimestampSec(); + int32_t delta = curSec - pMgmt->transSec; + if (delta > MNODE_TIMEOUT_SEC) { + mError("trans:%d, failed to propose since timeout, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, + pMgmt->transSec, curSec, delta, pMgmt->transSeq); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + terrno = TSDB_CODE_SYN_TIMEOUT; + pMgmt->errCode = TSDB_CODE_SYN_TIMEOUT; + tsem_post(&pMgmt->syncSem); + } else { + mDebug("trans:%d, waiting for sync confirm, start:%d cur:%d delta:%d seq:%" PRId64, pMgmt->transId, + pMgmt->transSec, curSec, curSec - pMgmt->transSec, pMgmt->transSeq); + } + } else { + // mTrace("check sync timeout msg, no trans waiting for confirm"); + } + taosThreadMutexUnlock(&pMgmt->lock); +} + int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { SSyncMgmt *pMgmt = &pMnode->syncMgmt; - pMgmt->errCode = 0; SRpcMsg req = {.msgType = TDMT_MND_APPLY_MSG, .contLen = sdbGetRawTotalSize(pRaw)}; if (req.contLen <= 0) return -1; @@ -314,38 +360,43 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { if (req.pCont == NULL) return -1; memcpy(req.pCont, pRaw, req.contLen); - taosWLockLatch(&pMgmt->lock); + taosThreadMutexLock(&pMgmt->lock); + pMgmt->errCode = 0; + if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); - taosWUnLockLatch(&pMgmt->lock); - terrno = TSDB_CODE_APP_NOT_READY; - return -1; + taosThreadMutexUnlock(&pMgmt->lock); + terrno = TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED; + return terrno; } mInfo("trans:%d, will be proposed", transId); pMgmt->transId = transId; - taosWUnLockLatch(&pMgmt->lock); + pMgmt->transSec = taosGetTimestampSec(); - int32_t code = syncPropose(pMgmt->sync, &req, false); + int64_t seq = 0; + int32_t code = syncPropose(pMgmt->sync, &req, false, &seq); if (code == 0) { - mInfo("trans:%d, is proposing and wait sem", pMgmt->transId); + mInfo("trans:%d, is proposing and wait sem, seq:%" PRId64, transId, seq); + pMgmt->transSeq = seq; + taosThreadMutexUnlock(&pMgmt->lock); tsem_wait(&pMgmt->syncSem); } else if (code > 0) { mInfo("trans:%d, confirm at once since replica is 1, continue execute", transId); - taosWLockLatch(&pMgmt->lock); pMgmt->transId = 0; - taosWUnLockLatch(&pMgmt->lock); + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + taosThreadMutexUnlock(&pMgmt->lock); sdbWriteWithoutFree(pMnode->pSdb, pRaw); sdbSetApplyInfo(pMnode->pSdb, req.info.conn.applyIndex, req.info.conn.applyTerm, SYNC_INDEX_INVALID); code = 0; } else { - mInfo("trans:%d, failed to proposed since %s", transId, terrstr()); - taosWLockLatch(&pMgmt->lock); + mError("trans:%d, failed to proposed since %s", transId, terrstr()); pMgmt->transId = 0; - taosWUnLockLatch(&pMgmt->lock); - if (terrno == TSDB_CODE_SYN_NOT_LEADER) { - terrno = TSDB_CODE_APP_NOT_READY; - } else { + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + taosThreadMutexUnlock(&pMgmt->lock); + if (terrno == 0) { terrno = TSDB_CODE_APP_ERROR; } } @@ -358,7 +409,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { } terrno = pMgmt->errCode; - return pMgmt->errCode; + return terrno; } void mndSyncStart(SMnode *pMnode) { @@ -371,30 +422,37 @@ void mndSyncStart(SMnode *pMnode) { } void mndSyncStop(SMnode *pMnode) { - taosWLockLatch(&pMnode->syncMgmt.lock); - if (pMnode->syncMgmt.transId != 0) { - mInfo("vgId:1, is stopped and post sem, trans:%d", pMnode->syncMgmt.transId); - pMnode->syncMgmt.transId = 0; - tsem_post(&pMnode->syncMgmt.syncSem); + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + mInfo("vgId:1, is stopped and post sem, trans:%d", pMgmt->transId); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->errCode = TSDB_CODE_APP_IS_STOPPING; + tsem_post(&pMgmt->syncSem); } - taosWUnLockLatch(&pMnode->syncMgmt.lock); + taosThreadMutexUnlock(&pMgmt->lock); } bool mndIsLeader(SMnode *pMnode) { + terrno = 0; SSyncState state = syncGetState(pMnode->syncMgmt.sync); - if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { - if (state.state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } - mDebug("vgId:1, mnode not ready, state:%s, restore:%d", syncStr(state.state), state.restored); + if (terrno != 0) { + mDebug("vgId:1, mnode is stopping"); + return false; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + mDebug("vgId:1, mnode not leader, state:%s", syncStr(state.state)); return false; } - if (!mndGetRestored(pMnode)) { - terrno = TSDB_CODE_APP_NOT_READY; + if (!state.restored || !pMnode->restored) { + terrno = TSDB_CODE_SYN_RESTORING; + mDebug("vgId:1, mnode not restored:%d:%d", state.restored, pMnode->restored); return false; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index eea74c6dd823faaa3467a87733db5ffe22528ac9..f25ebda38fe09bbfc34802016d9fc1f2152d288c 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -28,7 +28,7 @@ #include "parser.h" #include "tname.h" -#define MND_TOPIC_VER_NUMBER 1 +#define MND_TOPIC_VER_NUMBER 2 #define MND_TOPIC_RESERVE_SIZE 64 static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); @@ -93,6 +93,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { int32_t dataPos = 0; SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, pTopic->createUser, TSDB_USER_LEN, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); @@ -159,7 +160,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; - if (sver != MND_TOPIC_VER_NUMBER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto TOPIC_DECODE_OVER; } @@ -174,6 +175,9 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TOPIC_FNAME_LEN, TOPIC_DECODE_OVER); SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER); + if (sver >= 2) { + SDB_GET_BINARY(pRaw, dataPos, pTopic->createUser, TSDB_USER_LEN, TOPIC_DECODE_OVER); + } SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER); @@ -358,11 +362,18 @@ static int32_t extractTopicTbInfo(SNode *pAst, SMqTopicObj *pTopic) { return 0; } -static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb, + const char *userName) { mInfo("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(topicObj.createUser, userName, TSDB_USER_LEN); + + if (mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC, &topicObj) != 0) { + return -1; + } + topicObj.createTime = taosGetTimestampMs(); topicObj.updateTime = topicObj.createTime; topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); @@ -574,11 +585,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_TOPIC) != 0) { - goto _OVER; - } - - code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb, pReq->info.conn.user); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -634,7 +641,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } } - if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC) != 0) { + if (mndCheckTopicPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_TOPIC, pTopic) != 0) { mndReleaseTopic(pMnode, pTopic); return -1; } @@ -689,15 +696,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { sdbRelease(pSdb, pConsumer); } -#if 0 - if (pTopic->refConsumerCnt != 0) { - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); - return -1; - } -#endif - if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 27c58dfba163acbae22caa68cd3e6be7c492984c..d2fc2dc9b1ffed70f80a1571d22479e36e38d423 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -838,7 +838,7 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { return conflict; } -int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { +int32_t mndTrancCheckConflict(SMnode *pMnode, STrans *pTrans) { if (pTrans->conflict == TRN_CONFLICT_DB || pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { if (strlen(pTrans->dbname) == 0 && strlen(pTrans->stbname) == 0) { terrno = TSDB_CODE_MND_TRANS_CONFLICT; @@ -853,6 +853,14 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { return -1; } + return 0; +} + +int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { + if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + return -1; + } + if (taosArrayGetSize(pTrans->commitActions) <= 0) { terrno = TSDB_CODE_MND_TRANS_CLOG_IS_NULL; mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -918,10 +926,14 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && ((code == TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 60) || - (code != TSDB_CODE_APP_NOT_READY && pTrans->failedTimes > 6))) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION) { + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_SYN_PROPOSE_NOT_READY) { + if (pTrans->failedTimes > 60) sendRsp = true; + } else { + if (pTrans->failedTimes > 6) sendRsp = true; + } if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; - sendRsp = true; } } @@ -942,7 +954,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { code = TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL; } if (i != 0 && code == 0) { - code = TSDB_CODE_RPC_REDIRECT; + code = TSDB_CODE_MNODE_NOT_FOUND; } mInfo("trans:%d, client:%d send rsp, code:0x%x stage:%s app:%p", pTrans->id, i, code, mndTransStr(pTrans->stage), pInfo->ahandle); @@ -1024,6 +1036,7 @@ int32_t mndTransProcessRsp(SRpcMsg *pRsp) { if (pAction != NULL) { pAction->msgReceived = 1; pAction->errCode = pRsp->code; + pTrans->lastErrorNo = pRsp->code; } mInfo("trans:%d, %s:%d response is received, code:0x%x, accept:0x%x retry:0x%x", transId, mndTransStr(pAction->stage), @@ -1039,8 +1052,8 @@ static void mndTransResetAction(SMnode *pMnode, STrans *pTrans, STransAction *pA pAction->rawWritten = 0; pAction->msgSent = 0; pAction->msgReceived = 0; - if (pAction->errCode == TSDB_CODE_RPC_REDIRECT || pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || - pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { + if (pAction->errCode == TSDB_CODE_SYN_NEW_CONFIG_ERROR || pAction->errCode == TSDB_CODE_SYN_INTERNAL_ERROR || + pAction->errCode == TSDB_CODE_SYN_NOT_LEADER) { pAction->epSet.inUse = (pAction->epSet.inUse + 1) % pAction->epSet.numOfEps; mInfo("trans:%d, %s:%d execute status is reset and set epset inuse:%d", pTrans->id, mndTransStr(pAction->stage), pAction->id, pAction->epSet.inUse); @@ -1235,7 +1248,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (numOfActions == 0) return code; if (pTrans->redoActionPos >= numOfActions) return code; - mInfo("trans:%d, execute %d actions serial", pTrans->id, numOfActions); + mInfo("trans:%d, execute %d actions serial, current redoAction:%d", pTrans->id, numOfActions, pTrans->redoActionPos); for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); @@ -1286,13 +1299,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mInfo("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; - } else if (code == pAction->retryCode) { + } else if (code == pAction->retryCode || code == TSDB_CODE_SYN_PROPOSE_NOT_READY || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_SYN_NOT_LEADER) { mInfo("trans:%d, %s:%d receive code:0x%x and retry", pTrans->id, mndTransStr(pAction->stage), pAction->id, code); + pTrans->lastErrorNo = code; taosMsleep(300); action--; continue; } else { terrno = code; + pTrans->lastErrorNo = code; pTrans->code = code; mInfo("trans:%d, %s:%d receive code:0x%x and wait another schedule, failedTimes:%d", pTrans->id, mndTransStr(pAction->stage), pAction->id, code, pTrans->failedTimes); @@ -1321,6 +1337,7 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { } if (mndCannotExecuteTransAction(pMnode)) return false; + terrno = code; if (code == 0) { pTrans->code = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5f34adce7790112cd1f22917f81f4308e6369cf3..806ba0c98eee7237ea37c82a10b6eb4f5d6ea958 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -161,7 +161,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { SDB_SET_BINARY(pRaw, dataPos, topic, TSDB_TOPIC_FNAME_LEN, _OVER); - db = taosHashIterate(pUser->topics, topic); + topic = taosHashIterate(pUser->topics, topic); } SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) @@ -446,7 +446,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc return 0; } -SHashObj *mndDupDbHash(SHashObj *pOld) { +SHashObj *mndDupObjHash(SHashObj *pOld, int32_t dataLen) { SHashObj *pNew = taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pNew == NULL) { @@ -457,7 +457,7 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { char *db = taosHashIterate(pOld, NULL); while (db != NULL) { int32_t len = strlen(db) + 1; - if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) { + if (taosHashPut(pNew, db, len, db, dataLen) != 0) { taosHashCancelIterate(pOld, db); taosHashCleanup(pNew); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -469,6 +469,10 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { return pNew; } +SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAME_LEN); } + +SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); } + static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -519,7 +523,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); - newUser.topics = mndDupDbHash(pUser->topics); + newUser.topics = mndDupTopicHash(pUser->topics); taosRUnLockLatch(&pUser->lock); if (newUser.readDbs == NULL || newUser.writeDbs == NULL || newUser.topics == NULL) { @@ -832,6 +836,26 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfTopics = taosHashGetSize(pUser->topics); if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break; + if (pUser->superUser) { + cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(userName, pUser->user, pShow->pMeta->pSchemas[cols].bytes); + colDataAppend(pColInfo, numOfRows, (const char *)userName, false); + + char privilege[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(privilege, "all", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)privilege, false); + + char objName[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(objName, "all", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)objName, false); + + numOfRows++; + } + char *db = taosHashIterate(pUser->readDbs, NULL); while (db != NULL) { cols = 0; @@ -902,7 +926,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); numOfRows++; - db = taosHashIterate(pUser->topics, topic); + topic = taosHashIterate(pUser->topics, topic); } sdbRelease(pSdb, pUser); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 544215a905a6a48117885795ca0eebc62c605f32..31ab1f3259cd45b0223c5962fbca5f27386da57a 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -276,7 +276,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg } if (createReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -376,7 +376,7 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p } if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -507,7 +507,7 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup for (int32_t v = 0; v < pVgroup->replica; ++v) { SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pDnode = taosArrayGet(pArray, v); - if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } @@ -891,7 +891,7 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro } if (used) continue; - if (pDnode == NULL || pDnode->numOfVnodes > pDnode->numOfSupportVnodes) { + if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } @@ -998,7 +998,7 @@ int32_t mndAddCreateVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVg action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); @@ -1098,7 +1098,7 @@ int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgOb action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_NODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_VND_NOT_EXIST; if (isRedo) { if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1126,34 +1126,69 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, } if (!force) { - mInfo("vgId:%d, will add 1 vnode", pVgroup->vgId); - if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; - for (int32_t i = 0; i < newVg.replica - 1; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; - } - if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; - - mInfo("vgId:%d, will remove 1 vnode", pVgroup->vgId); - newVg.replica--; - SVnodeGid del = newVg.vnodeGid[vnIndex]; - newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; - memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); +#if 1 { - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); - if (pRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; +#else + if (newVg.replica == 1) { +#endif + mInfo("vgId:%d, will add 1 vnode, replca:%d", pVgroup->vgId, newVg.replica); + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; + for (int32_t i = 0; i < newVg.replica - 1; ++i) { + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + } + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[newVg.replica - 1]) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + + mInfo("vgId:%d, will remove 1 vnode, replca:2", pVgroup->vgId); + newVg.replica--; + SVnodeGid del = newVg.vnodeGid[vnIndex]; + newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; + memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); } - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - } - if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; - for (int32_t i = 0; i < newVg.replica; ++i) { - if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; + for (int32_t i = 0; i < newVg.replica; ++i) { + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + } + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; +#if 1 + } +#else + } else { // new replica == 3 + mInfo("vgId:%d, will add 1 vnode, replca:3", pVgroup->vgId); + if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; + mInfo("vgId:%d, will remove 1 vnode, replca:4", pVgroup->vgId); + newVg.replica--; + SVnodeGid del = newVg.vnodeGid[vnIndex]; + newVg.vnodeGid[vnIndex] = newVg.vnodeGid[newVg.replica]; + memset(&newVg.vnodeGid[newVg.replica], 0, sizeof(SVnodeGid)); + { + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) { + sdbFreeRaw(pRaw); + return -1; + } + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + } + + if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; + for (int32_t i = 0; i < newVg.replica; ++i) { + if (i == vnIndex) continue; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg, newVg.vnodeGid[i].dnodeId) != 0) return -1; + } + if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg, &newVg.vnodeGid[vnIndex]) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; } - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; +#endif } else { mInfo("vgId:%d, will add 1 vnode and force remove 1 vnode", pVgroup->vgId); if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg, pArray) != 0) return -1; diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 46a9a465ebe631665fa753c6de17569160a158d1..1f59a7fcca52497f9b95562eb3611df09ff31079 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 02_Alter_Acct) { @@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } TEST_F(MndTestAcct, 03_Drop_Acct) { @@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED); + ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT); } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 60be7cfbc03f2e19b1f8fd6f821eef08a7d56ee0..89c2b6931a321adf551516a685286149a7b003b9 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -173,7 +173,7 @@ class MndTestTrans2 : public ::testing::Test { action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED; mndTransAppendRedoAction(pTrans, &action); } @@ -190,7 +190,7 @@ class MndTestTrans2 : public ::testing::Test { action.pCont = pReq; action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; - action.acceptableCode = TSDB_CODE_NODE_ALREADY_DEPLOYED; + action.acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED; mndTransAppendUndoAction(pTrans, &action); } diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6558a98aaadbedca659268ff7b05f00243145450..318a746bb074425f0d086a9a99ce529bc0fbefa7 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -415,7 +415,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { break; } } else { - code = TSDB_CODE_SDB_APP_ERROR; + code = TSDB_CODE_APP_ERROR; taosHashCancelIterate(hash, ppRow); break; } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index c44b659ef589ed930fd9d92fb153fe3712311dca..32b34ea3a3b01193b0b23a2a2b4d80d938417a18 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -108,7 +108,7 @@ static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) { SHashObj *hash = pSdb->hashObjs[type]; if (hash == NULL) { - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -302,7 +302,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) { terrno = TSDB_CODE_SDB_OBJ_DROPPING; break; default: - terrno = TSDB_CODE_SDB_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; break; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7ef3207b4d61d8cf3b2edf11fe4ecc4821042a34..aa4d2fea4f03282dfd77f35c7699fd009fbe1b28 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -54,6 +54,7 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); +void vnodeSyncCheckTimeout(SVnode* pVnode); void vnodeClose(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); @@ -106,14 +107,24 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); -int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); -int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); -bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); -int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, - bool *acquired); -int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, - int32_t payloadLen, double selectivityRatio); -int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); + +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName); +int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid); +int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType); +bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid); +int32_t metaGetCachedTableUidList(SMeta *pMeta, tb_uid_t suid, const uint8_t *key, int32_t keyLen, SArray *pList, + bool *acquired); +int32_t metaUidFilterCachePut(SMeta *pMeta, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, + int32_t payloadLen, double selectivityRatio); +int32_t metaUidCacheClear(SMeta *pMeta, uint64_t suid); +tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name); +int64_t metaGetTbNum(SMeta *pMeta); +int64_t metaGetNtbNum(SMeta *pMeta); +typedef struct { + int64_t uid; + int64_t ctbNum; +} SMetaStbStats; +int32_t metaGetStbStats(SMeta *pMeta, int64_t uid, SMetaStbStats *pInfo); typedef struct SMetaFltParam { tb_uid_t suid; @@ -159,20 +170,19 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - STsdbReader **ppReader, const char *idstr); - -void tsdbReaderClose(STsdbReader *pReader); -bool tsdbNextDataBlock(STsdbReader *pReader); -bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid); -void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); -SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); -int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); -int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); -int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); -void *tsdbGetIdx(SMeta *pMeta); -void *tsdbGetIvtIdx(SMeta *pMeta); -uint64_t getReaderMaxVersion(STsdbReader *pReader); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); + +void tsdbReaderClose(STsdbReader *pReader); +bool tsdbNextDataBlock(STsdbReader *pReader); +void tsdbRetrieveDataBlockInfo(const STsdbReader *pReader, int32_t *rows, uint64_t *uid, STimeWindow *pWindow); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock* pDataBlock, bool *allHave); +SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); +int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); +int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); +int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); +uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void **pReader); @@ -212,11 +222,19 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - int64_t ver; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; + // const SSubmitReq *pMsg; + // SSubmitBlk *pBlock; + // SSubmitMsgIter msgIter; + // SSubmitBlkIter blkIter; + + int64_t ver; + SPackedData msg2; + + int8_t setMsg; + SSubmitReq2 submit; + int32_t nextBlk; + + int64_t lastBlkUid; SWalReader *pWalReader; @@ -241,11 +259,14 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); -int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); +int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas); +// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index dbfe9e1671c8a6728da61a80e95b185a9c42fc52..3999aa0b7f22a53197413048eca7842e6fe5e57d 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -70,7 +70,7 @@ int32_t metaCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo); int32_t metaStatsCacheDrop(SMeta* pMeta, int64_t uid); int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); -void metaUpdateStbStats(SMeta *pMeta, int64_t uid, int64_t delta); +void metaUpdateStbStats(SMeta* pMeta, int64_t uid, int64_t delta); int32_t metaUidFilterCacheGet(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, LRUHandle** pHandle); struct SMeta { @@ -79,7 +79,7 @@ struct SMeta { char* path; SVnode* pVnode; TDB* pEnv; - TXN txn; + TXN* txn; TTB* pTbDb; TTB* pSkmDb; TTB* pUidIdx; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 8ae1c8720ca6a37ba46e942febc0433dd091d282..0ef8e8bac592490dd1f8e48276b32b6effe52095 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -153,7 +153,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); +// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); @@ -181,7 +182,8 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink // void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +// void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 99adb6bf6a8faf45203b52e5f8034559e69903a8..0c473999851c7424210fb2c99fce26d2d11ea5e8 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -108,7 +108,7 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { #define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) #define TSDBROW_VERSION(ROW) \ (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) -#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->ts : -1) +#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1) #define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) #define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)}) #define tsdbRowFromBlockData(BLOCKDATA, IROW) \ diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e2b7327e8f7cea8851f70d5145b45e8f3a7c0a90..28797c536166d554aaa99f1d4866e76e85564971 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -77,7 +77,7 @@ void vnodeBufPoolReset(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); -void vnodeQueryPreClose(SVnode *pVnode); +void vnodeQueryPreClose(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct); int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct); @@ -86,7 +86,6 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); int32_t vnodeShouldCommit(SVnode* pVnode); -int32_t vnodeCommit(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo); @@ -100,7 +99,7 @@ int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncClose(SVnode* pVnode); -void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg); +void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index eb11e4ac22629ddd73d2e732843421feae5c5897..603a68ac6c448e839a4265c8f23760992bea14fc 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -75,6 +75,7 @@ typedef struct SStreamStateWriter SStreamStateWriter; typedef struct SRSmaSnapReader SRSmaSnapReader; typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; +typedef struct SCommitInfo SCommitInfo; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -86,23 +87,32 @@ typedef struct SSnapDataHdr SSnapDataHdr; #define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA2_DIR "rsma2" +#define VND_INFO_FNAME "vnode.json" + // vnd.h void* vnodeBufPoolMalloc(SVBufPool* pPool, int32_t size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); void vnodeBufPoolRef(SVBufPool* pPool); void vnodeBufPoolUnRef(SVBufPool* pPool); +int vnodeDecodeInfo(uint8_t* pData, SVnodeInfo* pInfo); // meta typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; +#define META_BEGIN_HEAP_BUFFERPOOL 0 +#define META_BEGIN_HEAP_OS 1 +#define META_BEGIN_HEAP_NIL 2 + int metaOpen(SVnode* pVnode, SMeta** ppMeta, int8_t rollback); int metaClose(SMeta* pMeta); int metaBegin(SMeta* pMeta, int8_t fromSys); -int metaCommit(SMeta* pMeta); -int metaFinishCommit(SMeta* pMeta); +TXN* metaGetTxn(SMeta* pMeta); +int metaCommit(SMeta* pMeta, TXN* txn); +int metaFinishCommit(SMeta* pMeta, TXN* txn); int metaPrepareAsyncCommit(SMeta* pMeta); +int metaAbort(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); @@ -116,8 +126,6 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); -tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); -int64_t metaGetTbNum(SMeta* pMeta); int64_t metaGetTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock); void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock); @@ -144,22 +152,17 @@ typedef struct SMetaInfo { } SMetaInfo; int32_t metaGetInfo(SMeta* pMeta, int64_t uid, SMetaInfo* pInfo, SMetaReader* pReader); -typedef struct { - int64_t uid; - int64_t ctbNum; -} SMetaStbStats; -int32_t metaGetStbStats(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo); - // tsdb int tsdbOpen(SVnode* pVnode, STsdb** ppTsdb, const char* dir, STsdbKeepCfg* pKeepCfg, int8_t rollback); int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); -int32_t tsdbCommit(STsdb* pTsdb); +int32_t tsdbPrepareCommit(STsdb* pTsdb); +int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); @@ -185,7 +188,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); @@ -197,9 +200,9 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq); +int32_t tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, + SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, + SBatchDeleteReq* pDeleteReq, void** ppData, int32_t* pLen); // sma int32_t smaInit(); @@ -210,8 +213,8 @@ int32_t smaBegin(SSma* pSma); int32_t smaSyncPreCommit(SSma* pSma); int32_t smaSyncCommit(SSma* pSma); int32_t smaSyncPostCommit(SSma* pSma); -int32_t smaPreCommit(SSma* pSma); -int32_t smaCommit(SSma* pSma); +int32_t smaPrepareAsyncCommit(SSma* pSma); +int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); int32_t smaPostCommit(SSma* pSma); int32_t smaDoRetention(SSma* pSma, int64_t now); @@ -220,7 +223,7 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); -int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaSubmit(SSma* pSma, int64_t version, void* pReq, void* pMsg, int32_t len, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); @@ -242,6 +245,7 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData); // STsdbSnapWriter ======================================== int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter); int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter); int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback); // STqSnapshotReader == int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapReader** ppReader); @@ -348,7 +352,12 @@ struct SVnode { bool blocked; bool restored; tsem_t syncSem; + int32_t blockSec; + int64_t blockSeq; SQHandle* pQuery; +#if 0 + SRpcHandleInfo blockInfo; +#endif }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) @@ -413,6 +422,12 @@ struct SSnapDataHdr { uint8_t data[]; }; +struct SCommitInfo { + SVnodeInfo info; + SVnode* pVnode; + TXN* txn; +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 6a704d042577391ea382fc84e979f124e6250c62..37dcec4f85a973c10e14f513245d3fa59287d3c7 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -32,9 +32,9 @@ typedef struct SMetaStbStatsEntry { } SMetaStbStatsEntry; typedef struct STagFilterResEntry { - uint64_t suid; // uid for super table - SList list; // the linked list of md5 digest, extracted from the serialized tag query condition - uint32_t qTimes;// queried times for current super table + uint64_t suid; // uid for super table + SList list; // the linked list of md5 digest, extracted from the serialized tag query condition + uint32_t qTimes; // queried times for current super table } STagFilterResEntry; struct SMetaCache { @@ -126,13 +126,14 @@ int32_t metaCacheOpen(SMeta* pMeta) { goto _err2; } - pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5*1024*1024, -1, 0.5); + pCache->sTagFilterResCache.pUidResCache = taosLRUCacheInit(5 * 1024 * 1024, -1, 0.5); if (pCache->sTagFilterResCache.pUidResCache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; } - pCache->sTagFilterResCache.pTableEntry = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); + pCache->sTagFilterResCache.pTableEntry = + taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); if (pCache->sTagFilterResCache.pTableEntry == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err2; @@ -419,7 +420,8 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { return code; } -int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, bool* acquireRes) { +int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pKey, int32_t keyLen, SArray* pList1, + bool* acquireRes) { uint64_t* pBuf = pMeta->pCache->sTagFilterResCache.keyBuf; // generate the composed key for LRU cache @@ -428,8 +430,8 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK pBuf[0] = suid; memcpy(&pBuf[1], pKey, keyLen); - int32_t len = keyLen + sizeof(uint64_t); - LRUHandle *pHandle = taosLRUCacheLookup(pCache, pBuf, len); + int32_t len = keyLen + sizeof(uint64_t); + LRUHandle* pHandle = taosLRUCacheLookup(pCache, pBuf, len); if (pHandle == NULL) { *acquireRes = 0; return TSDB_CODE_SUCCESS; @@ -439,7 +441,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK *acquireRes = 1; const char* p = taosLRUCacheValue(pMeta->pCache->sTagFilterResCache.pUidResCache, pHandle); - int32_t size = *(int32_t*) p; + int32_t size = *(int32_t*)p; taosArrayAddBatch(pList1, p + sizeof(int32_t), size); (*pEntry)->qTimes += 1; @@ -454,7 +456,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK SListNode* pNode = NULL; while ((pNode = tdListNext(&iter)) != NULL) { - memcpy(pBuf + sizeof(suid), pNode->data, keyLen); + memcpy(&pBuf[1], pNode->data, keyLen); // check whether it is existed in LRU cache, and remove it from linked list if not. LRUHandle* pRes = taosLRUCacheLookup(pCache, pBuf, len); @@ -467,12 +469,15 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK // remove the keys, of which query uid lists have been replaced already. size_t s = taosArrayGetSize(pList); - for(int32_t i = 0; i < s; ++i) { + for (int32_t i = 0; i < s; ++i) { SListNode** p1 = taosArrayGet(pList, i); tdListPopNode(&(*pEntry)->list, *p1); + taosMemoryFree(*p1); } - (*pEntry)->qTimes = 0; // reset the query times + (*pEntry)->qTimes = 0; // reset the query times + + taosArrayDestroy(pList); } } @@ -487,7 +492,8 @@ static void freePayload(const void* key, size_t keyLen, void* value) { } // check both the payload size and selectivity ratio -int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, int32_t payloadLen, double selectivityRatio) { +int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int32_t keyLen, void* pPayload, + int32_t payloadLen, double selectivityRatio) { if (selectivityRatio > tsSelectivityRatio) { metaDebug("vgId:%d, suid:%" PRIu64 " failed to add to uid list cache, due to selectivity ratio %.2f less than threshold %.2f", @@ -525,9 +531,10 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int ASSERT(sizeof(uint64_t) + keyLen == 24); // add to cache. - taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW); - metaDebug("vgId:%d, suid:%"PRIu64" list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), - suid, (int32_t) taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); + taosLRUCacheInsert(pCache, pBuf, sizeof(uint64_t) + keyLen, pPayload, payloadLen, freePayload, NULL, + TAOS_LRU_PRIORITY_LOW); + metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, + (int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry)); return TSDB_CODE_SUCCESS; } @@ -539,7 +546,7 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } - int32_t keyLen = sizeof(uint64_t) * 3; + int32_t keyLen = sizeof(uint64_t) * 3; uint64_t p[3] = {0}; p[0] = suid; diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index 0be0c3e40798ec953a529d433b7294d564ceee84..ac8d99ccf0afbbf1074732ae75fc54e515122f80 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -19,13 +19,22 @@ static FORCE_INLINE void *metaMalloc(void *pPool, size_t size) { return vnodeBuf static FORCE_INLINE void metaFree(void *pPool, void *p) { vnodeBufPoolFree((SVBufPool *)pPool, p); } // begin a meta txn -int metaBegin(SMeta *pMeta, int8_t fromSys) { - if (fromSys) { - tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - } else { - tdbTxnOpen(&pMeta->txn, 0, metaMalloc, metaFree, pMeta->pVnode->inUse, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); +int metaBegin(SMeta *pMeta, int8_t heap) { + void *(*xMalloc)(void *, size_t) = NULL; + void (*xFree)(void *, void *) = NULL; + void *xArg = NULL; + + // default heap to META_BEGIN_HEAP_NIL + if (heap == META_BEGIN_HEAP_OS) { + xMalloc = tdbDefaultMalloc; + xFree = tdbDefaultFree; + } else if (heap == META_BEGIN_HEAP_BUFFERPOOL) { + xMalloc = metaMalloc; + xFree = metaFree; + xArg = pMeta->pVnode->inUse; } - if (tdbBegin(pMeta->pEnv, &pMeta->txn) < 0) { + + if (tdbBegin(pMeta->pEnv, &pMeta->txn, xMalloc, xFree, xArg, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } @@ -33,9 +42,16 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { } // commit the meta txn -int metaCommit(SMeta *pMeta) { return tdbCommit(pMeta->pEnv, &pMeta->txn); } -int metaFinishCommit(SMeta *pMeta) { return tdbPostCommit(pMeta->pEnv, &pMeta->txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, &pMeta->txn); } +TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } +int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } +int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { + // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); + int code = 0; + code = tdbCommit(pMeta->pEnv, pMeta->txn); + + return code; +} // abort the meta txn -int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, &pMeta->txn); } +int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, pMeta->txn); } diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 6d0dac88fb89478842bcd278ea44a791d7e7b152..619ae03ae4af99dc14565accea950b25fd28ed38 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -223,6 +223,23 @@ int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } + +int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { + int code = 0; + SMetaReader mr = {0}; + metaReaderInit(&mr, (SMeta *)meta, 0); + code = metaGetTableEntryByUid(&mr, uid); + if (code < 0) { + metaReaderClear(&mr); + return -1; + } + strncpy(tbName, mr.me.name, TSDB_TABLE_NAME_LEN); + metaReaderClear(&mr); + + return 0; +} + + int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; @@ -720,6 +737,10 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } +int64_t metaGetNtbNum(SMeta *pMeta) { + return pMeta->pVnode->config.vndStats.numOfNTables; +} + typedef struct { SMeta *pMeta; TBC *pCur; diff --git a/source/dnode/vnode/src/meta/metaSma.c b/source/dnode/vnode/src/meta/metaSma.c index 52452bf710745d4da4ec1f3d52dea00ece97f160..8d5821e28bfa2c33a5582e34bc32127970af6320 100644 --- a/source/dnode/vnode/src/meta/metaSma.c +++ b/source/dnode/vnode/src/meta/metaSma.c @@ -117,7 +117,7 @@ static int metaSaveSmaToDB(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { goto _err; } @@ -131,17 +131,17 @@ _err: static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { SUidIdxVal uidIdxVal = {.suid = pME->smaEntry.tsma->indexUid, .version = pME->version, .skmVer = 0}; - return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn); + return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } static int metaUpdateSmaIdx(SMeta *pMeta, const SMetaEntry *pME) { SSmaIdxKey smaIdxKey = {.uid = pME->smaEntry.tsma->tableUid, .smaUid = pME->smaEntry.tsma->indexUid}; - return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSmaIdx, &smaIdxKey, sizeof(smaIdxKey), NULL, 0, pMeta->txn); } static int metaHandleSmaEntry(SMeta *pMeta, const SMetaEntry *pME) { diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 5c5b49ece57bd2a2a9b787c356108fc108cee4a4..974f8a92181317db061bbdb0e1c4ad7bfae0dc40 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -145,7 +145,7 @@ int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWr pWriter->sver = sver; pWriter->ever = ever; - metaBegin(pMeta, 1); + metaBegin(pMeta, META_BEGIN_HEAP_NIL); *ppWriter = pWriter; return code; @@ -161,11 +161,12 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) { SMetaSnapWriter* pWriter = *ppWriter; if (rollback) { - ASSERT(0); + code = metaAbort(pWriter->pMeta); + if (code) goto _err; } else { - code = metaCommit(pWriter->pMeta); + code = metaCommit(pWriter->pMeta, pWriter->pMeta->txn); if (code) goto _err; - code = metaFinishCommit(pWriter->pMeta); + code = metaFinishCommit(pWriter->pMeta, pWriter->pMeta->txn); if (code) goto _err; } taosMemoryFree(pWriter); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 5c97ee56337061ae2dcf5e2159ea33bd4dd323b3..722a93137c256be4e91338c72f1fcd89521ce310 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -261,7 +261,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb // drop all child tables TBC *pCtbIdxc = NULL; - tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = pReq->suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); if (rc < 0) { tdbTbcClose(pCtbIdxc); @@ -295,10 +295,10 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb _drop_super_table: tdbTbGet(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pData, &nData); tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = ((SUidIdxVal *)pData)[0].version, .uid = pReq->suid}, - sizeof(STbDbKey), &pMeta->txn); - tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn); - tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); - tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); + sizeof(STbDbKey), pMeta->txn); + tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, pMeta->txn); + tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), pMeta->txn); metaULock(pMeta); @@ -321,7 +321,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int32_t ret; int32_t c = -2; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); if (ret < 0 || c) { tdbTbcClose(pUidIdxc); @@ -340,7 +340,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { oversion = ((SUidIdxVal *)pData)[0].version; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(ret == 0 && c == 0); @@ -549,8 +549,8 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { } static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { - int64_t ttlDays; - int64_t ctime; + int64_t ttlDays = 0; + int64_t ctime = 0; if (pME->type == TSDB_CHILD_TABLE) { ctime = pME->ctbEntry.ctime; ttlDays = pME->ctbEntry.ttlDays; @@ -595,7 +595,7 @@ static int metaDeleteTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { STtlIdxKey ttlKey = {0}; metaBuildTtlIdxKey(&ttlKey, pME); if (ttlKey.dtime == 0) return 0; - return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), &pMeta->txn); + return tdbTbDelete(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), pMeta->txn); } static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { @@ -657,7 +657,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, &pTagIdxKey, &nTagIdxKey) == 0) { - tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, &pMeta->txn); + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); } @@ -667,9 +667,9 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { } } - tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pMeta->txn); - tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, &pMeta->txn); - tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); + tdbTbDelete(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), pMeta->txn); + tdbTbDelete(pMeta->pNameIdx, e.name, strlen(e.name) + 1, pMeta->txn); + tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), pMeta->txn); if (e.type == TSDB_CHILD_TABLE || e.type == TSDB_NORMAL_TABLE) metaDeleteCtimeIdx(pMeta, &e); if (e.type == TSDB_NORMAL_TABLE) metaDeleteNcolIdx(pMeta, &e); @@ -677,7 +677,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { if (e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); if (e.type == TSDB_CHILD_TABLE) { - tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); + tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), pMeta->txn); --pMeta->pVnode->config.vndStats.numOfCTables; @@ -689,7 +689,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { --pMeta->pVnode->config.vndStats.numOfNTables; pMeta->pVnode->config.vndStats.numOfNTimeSeries -= e.ntbEntry.schemaRow.nCols - 1; } else if (e.type == TSDB_SUPER_TABLE) { - tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), pMeta->txn); // drop schema.db (todo) metaStatsCacheDrop(pMeta, uid); @@ -710,7 +710,7 @@ int metaUpdateCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { return 0; } - return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), NULL, 0, pMeta->txn); } int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -718,14 +718,14 @@ int metaDeleteCtimeIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildCtimeIdxKey(&ctimeKey, pME) < 0) { return 0; } - return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), &pMeta->txn); + return tdbTbDelete(pMeta->pCtimeIdx, &ctimeKey, sizeof(ctimeKey), pMeta->txn); } int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { SNcolIdxKey ncolKey = {0}; if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { return 0; } - return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), NULL, 0, pMeta->txn); } int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { @@ -733,7 +733,7 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) { if (metaBuildNColIdxKey(&ncolKey, pME) < 0) { return 0; } - return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), &pMeta->txn); + return tdbTbDelete(pMeta->pNcolIdx, &ncolKey, sizeof(ncolKey), pMeta->txn); } static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { @@ -768,7 +768,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -778,7 +778,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl // search table.db TBC *pTbDbc = NULL; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -959,7 +959,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -972,7 +972,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA SDecoder dc2 = {0}; /* get ctbEntry */ - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -1075,7 +1075,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, - ((STag *)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); metaUidCacheClear(pMeta, ctbEntry.ctbEntry.suid); @@ -1125,7 +1125,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search uid index TBC *pUidIdxc = NULL; - tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); + tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); ASSERT(c == 0); @@ -1135,7 +1135,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p // search table.db TBC *pTbDbc = NULL; - tdbTbcOpen(pMeta->pTbDb, &pTbDbc, &pMeta->txn); + tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); ASSERT(c == 0); tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); @@ -1242,7 +1242,7 @@ static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderClear(&coder); // write to table.db - if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pTbDb, pKey, kLen, pVal, vLen, pMeta->txn) < 0) { goto _err; } @@ -1265,29 +1265,29 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { SUidIdxVal uidIdxVal = {.suid = info.suid, .version = info.version, .skmVer = info.skmVer}; - return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), &pMeta->txn); + return tdbTbUpsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &uidIdxVal, sizeof(uidIdxVal), pMeta->txn); } static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, pMeta->txn); } static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { - return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); + return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), pMeta->txn); } static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { STtlIdxKey ttlKey = {0}; metaBuildTtlIdxKey(&ttlKey, pME); if (ttlKey.dtime == 0) return 0; - return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pTtlIdx, &ttlKey, sizeof(ttlKey), NULL, 0, pMeta->txn); } static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, - ((STag *)(pME->ctbEntry.pTags))->len, &pMeta->txn); + ((STag *)(pME->ctbEntry.pTags))->len, pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, @@ -1353,6 +1353,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } + if (stbEntry.stbEntry.schemaTag.pSchema == NULL) { + goto end; + } + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; STagVal tagVal = {.cid = pTagColumn->colId}; @@ -1379,7 +1383,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { ret = -1; goto end; } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, &pMeta->txn); + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } end: metaDestroyTagIdxKey(pTagIdxKey); @@ -1426,7 +1430,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { tEncoderInit(&coder, pVal, vLen); tEncodeSSchemaWrapper(&coder, pSW); - if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, &pMeta->txn) < 0) { + if (tdbTbInsert(pMeta->pSkmDb, &skmDbKey, sizeof(skmDbKey), pVal, vLen, pMeta->txn) < 0) { rcode = -1; goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index a79ae35d79f2725d0765cd5bcee9108b178c57f0..9748963722cb5dac2f0a83bc89fad9a04987cf28 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -23,7 +23,7 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); #endif static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); @@ -59,7 +59,7 @@ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(p * @param pSma * @return int32_t */ -int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } +int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } /** * @brief async commit, only applicable to Rollup SMA @@ -67,7 +67,7 @@ int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); * @param pSma * @return int32_t */ -int32_t smaCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } +int32_t smaCommit(SSma *pSma, SCommitInfo *pInfo) { return tdProcessRSmaAsyncCommitImpl(pSma, pInfo); } /** * @brief async commit, only applicable to Rollup SMA @@ -127,8 +127,8 @@ _exit: } int32_t smaFinishCommit(SSma *pSma) { - int32_t code = 0; - SVnode *pVnode = pSma->pVnode; + int32_t code = 0; + SVnode *pVnode = pSma->pVnode; if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { smaError("vgId:%d, failed to finish commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); @@ -378,6 +378,11 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); #endif + // all rsma results are written completely + STsdb *pTsdb = NULL; + if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + return TSDB_CODE_SUCCESS; } @@ -387,9 +392,9 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { - int32_t code = 0; - SVnode *pVnode = pSma->pVnode; +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { + int32_t code = 0; + SVnode *pVnode = pSma->pVnode; #if 0 SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); @@ -399,11 +404,11 @@ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { } #endif - if ((code = tsdbCommit(VND_RSMA1(pVnode))) < 0) { + if ((code = tsdbCommit(VND_RSMA1(pVnode), pInfo)) < 0) { smaError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); goto _exit; } - if ((code = tsdbCommit(VND_RSMA2(pVnode))) < 0) { + if ((code = tsdbCommit(VND_RSMA2(pVnode), pInfo)) < 0) { smaError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); goto _exit; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 6bd2ae3435349dfc3ab81838424cccc06ec4811e..77cc78e47b091334c37d9200401d75d1d6dab228 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -601,8 +601,8 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_FAILED; } - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - // TODO: spin lock for race conditiond + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -610,29 +610,19 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; +static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { + SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; + int32_t size = taosArrayGetSize(pSubmitTbData); terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); + if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { return -1; } - - if (!pBlock) break; - tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } return 0; } @@ -708,14 +698,14 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId, - output->info.rows); + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + output->info.id.groupId, output->info.rows); - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); - SSubmitReq *pReq = NULL; + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { + if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); @@ -723,19 +713,21 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - taosMemoryFreeClear(pReq); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); goto _err; } - smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64 - " len %" PRIu32, - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version, - htonl(pReq->header.contLen)); + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, + SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); - taosMemoryFreeClear(pReq); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } } } @@ -753,22 +745,29 @@ _err: * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma + * @param version * @param pMsg + * @param len * @param inputType * @param pInfo * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid) { - const SSubmitReq *pReq = (const SSubmitReq *)pMsg; +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t size = sizeof(int32_t) + sizeof(int64_t) + len; + void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); - void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM); if (!qItem) { return TSDB_CODE_FAILED; } - memcpy(qItem, pMsg, pReq->header.contLen); + void *pItem = qItem; + + *(int32_t *)pItem = len; + pItem = POINTER_SHIFT(pItem, sizeof(int32_t)); + *(int64_t *)pItem = version; + memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); taosWriteQitem(pInfo->queue, qItem); @@ -1010,12 +1009,14 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { * @brief async mode * * @param pSma + * @param version * @param pMsg * @param inputType * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -1023,7 +1024,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -1045,7 +1046,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { +int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists @@ -1059,19 +1060,22 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); goto _err; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, uidStore.suid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); goto _err; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); goto _err; } } @@ -1081,7 +1085,6 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1378,7 +1381,8 @@ _end: static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { - taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + SPackedData *packData = taosArrayGet(pItems, i); + taosFreeQitem(POINTER_SHIFT(packData->msgStr, -sizeof(int32_t) - sizeof(int64_t))); } taosArrayClear(pItems); } @@ -1447,7 +1451,11 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (!taosArrayPush(pSubmitArr, &msg)) { + SPackedData packData = {.msgLen = *(int32_t *)msg, + .ver = *(int64_t *)POINTER_SHIFT(msg, sizeof(int32_t)), + .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; + + if (!taosArrayPush(pSubmitArr, &packData)) { tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } @@ -1502,7 +1510,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } if (!(pSubmitArr = - taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) { + taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), sizeof(SPackedData)))) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index a2c9484693a720243ccb205f0d311289c86f254e..cf1e5d5d397df0e5de2c41e15c9dd92adbcfabe0 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -204,18 +204,18 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq = {0}; - SSubmitReq *pSubmitReq = - tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); - + void *pSubmitReq = NULL; + int32_t contLen = 0; - if (!pSubmitReq) { - smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + if (tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen) < 0) { + smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); goto _err; } + // TODO deleteReq + taosArrayDestroy(deleteReq.deleteReqs); + #if 0 ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)); @@ -224,7 +224,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, .pCont = pSubmitReq, - .contLen = ntohl(pSubmitReq->length), + .contLen = ntohl(contLen), }; if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 04cf8bacc029a4d9e033410d42c67ebba64534ba..fdaedd795bdfa2ef563f8840607c6ba9612d0073 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -75,7 +75,7 @@ static void tqPushEntryFree(void* data) { STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); @@ -673,9 +673,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) { + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SMsgHead)), + .msgLen = pHead->bodyLen - sizeof(SMsgHead), + .ver = pHead->version, + }; + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); @@ -725,18 +728,25 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgL } taosWUnLockLatch(&pTq->pushLock); - code = taosHashRemove(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); - if (code != 0) { - tqError("cannot process tq delete req %s, since no such handle", pReq->subKey); + STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); + if (pHandle) { + //walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); + if (pHandle->pRef) { + walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); + } + code = taosHashRemove(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); + if (code != 0) { + tqError("cannot process tq delete req %s, since no such handle", pReq->subKey); + } } code = tqOffsetDelete(pTq->pOffsetStore, pReq->subKey); if (code != 0) { - tqError("cannot process tq delete req %s, since no such offset", pReq->subKey); + tqError("cannot process tq delete req %s, since no such offset in cache", pReq->subKey); } if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) { - ASSERT(0); + tqError("cannot process tq delete req %s, since no such offset in tdb", pReq->subKey); } return 0; } @@ -944,7 +954,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; ASSERT(pTask->tbSink.pSchemaWrapper); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); @@ -1265,7 +1275,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); if (!failed) { - SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM); + SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); pRefBlock->type = STREAM_INPUT__REF_DATA_BLOCK; pRefBlock->pBlock = pDelBlock; pRefBlock->dataRef = pRef; @@ -1297,7 +1307,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } #if 0 - SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); SSDataBlock block = {0}; @@ -1325,12 +1335,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit* pSubmit = NULL; +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { + void* pIter = NULL; + bool failed = false; + SStreamDataSubmit2* pSubmit = NULL; - pSubmit = streamDataSubmitNew(pReq); + pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1347,7 +1357,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 2514190035d5b402d3b7a4565284ac968ea61631..1074e2f6d5abaac724bfeba6e7d9f33ffe248a35 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -25,7 +25,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t pRetrieve->precision = precision; pRetrieve->compressed = 0; pRetrieve->completed = 1; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); @@ -110,14 +110,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } ASSERT(pRsp->rspOffset.type != 0); - if (pRsp->withTbName) { - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, 1); - } else { - pRsp->withTbName = false; - } - } + ASSERT(pRsp->withTbName == false); ASSERT(pRsp->withSchema == false); return 0; @@ -154,9 +147,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL) { if (pRsp->withTbName) { - int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { - uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { continue; } @@ -223,7 +215,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { STqExecHandle* pExec = &pHandle->execHandle; ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN); @@ -232,8 +224,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlock(pReader)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock2(pReader)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ @@ -241,11 +234,12 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -255,7 +249,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; + int64_t uid = pExec->pExecReader->lastBlkUid; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { if (pRsp->createTableNum == 0) { @@ -268,6 +264,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); @@ -281,19 +278,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; + int64_t uid = pExec->pExecReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -303,6 +301,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } if (pHandle->fetchMeta) { +#if 0 SSubmitBlk* pBlk = pReader->pBlock; int32_t schemaLen = htonl(pBlk->schemaLen); if (schemaLen > 0) { @@ -316,6 +315,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayPush(pRsp->createTableReq, &createReq); pRsp->createTableNum++; } +#endif } /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ /*pTq->pVnode->config.tsdbCfg.precision);*/ diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 612e465295fafa0843a941fe7c1fe73c3002fb3d..f476f58b565943f5cdd1e8c99233e37a9f1fdf24 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -108,24 +108,22 @@ int32_t tqMetaClose(STQ* pTq) { } int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen) { - TXN txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; - } + TXN* txn; - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { return -1; } - if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, &txn) < 0) { + if (tdbTbUpsert(pTq->pCheckStore, key, strlen(key), value, vLen, txn) < 0) { return -1; } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { return -1; } - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { return -1; } @@ -133,25 +131,22 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ } int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { - TXN txn; - - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); - } + TXN* txn; - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), &txn) < 0) { + if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { /*ASSERT(0);*/ } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } @@ -219,25 +214,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { ASSERT(0); } - TXN txn; + TXN* txn; - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, txn) < 0) { ASSERT(0); } - if (tdbTbUpsert(pTq->pExecStore, key, (int)strlen(key), buf, vlen, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { - ASSERT(0); - } - - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } @@ -247,25 +239,22 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { } int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { - TXN txn; - - if (tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); - } + TXN* txn; - if (tdbBegin(pTq->pMetaDB, &txn) < 0) { + if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < + 0) { ASSERT(0); } - if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), &txn) < 0) { + if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { /*ASSERT(0);*/ } - if (tdbCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } - if (tdbPostCommit(pTq->pMetaDB, &txn) < 0) { + if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index f89bc2036252961ff10d81774a9c6f756fdebf43..948e037c97427f5c35cf956604a91eb47f1aac44 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -213,7 +213,11 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s", pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType)); + void* pReq = POINTER_SHIFT(msg, sizeof(SMsgHead)); + int32_t len = msgLen - sizeof(SMsgHead); + + tqDebug("vgId:%d, tq push msg ver %" PRId64 ", type: %s, p head %p, p body %p, len %d", pTq->pVnode->config.vgId, ver, + TMSG_INFO(msgType), msg, pReq, len); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost @@ -222,7 +226,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (taosHashGetSize(pTq->pPushMgr) != 0) { SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); @@ -230,9 +234,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosArrayDestroy(cachedKeyLens); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); void* pIter = NULL; while (1) { @@ -256,7 +258,12 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) SMqDataRsp* pRsp = &pPushEntry->dataRsp; // prepare scan mem data - qStreamScanMemData(task, pReq); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + qStreamSetScanMemData(task, submit); // exec while (1) { @@ -310,17 +317,22 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + + tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); - tqProcessSubmitReq(pTq, data, ver); + tqProcessSubmitReq(pTq, submit); } if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index c3a4cefc66bdfc3c7e6e38a5d8cd1f92c6bd42bf..e662a7daadc6ae9ea7ca998de5fa8e4b62c462ce 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -252,7 +252,7 @@ END: } STqReader* tqOpenReader(SVnode* pVnode) { - STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; } @@ -264,7 +264,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - pReader->pMsg = NULL; + /*pReader->pMsg = NULL;*/ pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; @@ -306,7 +306,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver) { } int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->pMsg != NULL; + bool fromProcessedMsg = pReader->msg2.msgStr != NULL; while (1) { if (!fromProcessedMsg) { @@ -320,7 +320,9 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { ASSERT(ret->offset.version >= 0); return -1; } - void* body = pReader->pWalReader->pHead->head.body; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SMsgHead)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SMsgHead); + int64_t ver = pReader->pWalReader->pHead->head.version; #if 0 if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { // TODO do filter @@ -329,16 +331,17 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return 0; } else { #endif - tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + /*tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);*/ #if 0 } #endif } - while (tqNextDataBlock(pReader)) { + while (tqNextDataBlock2(pReader)) { // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + int32_t code = tqRetrieveDataBlock2(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { ASSERT(0); continue; @@ -359,6 +362,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } } +#if 0 int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { pReader->pMsg = pMsg; @@ -373,7 +377,33 @@ int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t v memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } +#endif + +int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { + ASSERT(pReader->msg2.msgStr == NULL); + ASSERT(msgStr); + ASSERT(msgLen); + ASSERT(ver >= 0); + pReader->msg2.msgStr = msgStr; + pReader->msg2.msgLen = msgLen; + pReader->msg2.ver = ver; + pReader->ver = ver; + + tqDebug("tq reader set msg %p %d", msgStr, msgLen); + if (pReader->setMsg == 0) { + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + pReader->setMsg = 1; + } + return 0; +} + +#if 0 bool tqNextDataBlock(STqReader* pReader) { if (pReader->pMsg == NULL) return false; while (1) { @@ -397,6 +427,59 @@ bool tqNextDataBlock(STqReader* pReader) { } return false; } +#endif + +bool tqNextDataBlock2(STqReader* pReader) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + ASSERT(pSubmitTbData->uid); + + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} + +bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret == NULL) { + return true; + } + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { int32_t code; @@ -427,6 +510,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +#if 0 bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -443,6 +527,253 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { return false; } +int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { + // + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + } + return 0; +} +#endif + +int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader) { + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + ASSERT(pReader->nextBlk < blockSz); + + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg2.ver; + + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + + if (colNumNeed == 0) { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + colNeed++; + } + } + } + + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + ASSERT(sourceIdx < numOfCols); + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + ASSERT(pCol->nVal == numOfRows); + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, sourceIdx, &colVal); +#if 0 + void* val = NULL; + if (IS_STR_DATA_TYPE(colVal.type)) { + val = colVal.value.pData; + } else { + val = &colVal.value.val; + } + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } +#endif + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + ASSERT(0); + } + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + ASSERT(sourceIdx < pTschema->numOfCols); + + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataAppendNULL(pColData, i); + } + /*val = colVal.value.pData;*/ + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + + sourceIdx++; + targetIdx++; + break; + } else { + ASSERT(0); + } + } + } + } + } + } + + return 0; + +FAIL: + blockDataFreeRes(pBlock); + return -1; +} + +#if 0 int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema int32_t sversion = htonl(pReader->pBlock->sversion); @@ -533,6 +864,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { pBlock->info.id.uid = pReader->msgIter.uid; pBlock->info.rows = pReader->msgIter.numOfRows; pBlock->info.version = pReader->pMsg->version; + pBlock->info.dataLoad = 1; while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); @@ -690,6 +1022,18 @@ FAIL: taosMemoryFree(assigned); return -1; } +#endif + +int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas) { + SSDataBlock block = {0}; + if (tqRetrieveDataBlock2(&block, pReader) == 0) { + taosArrayPush(blocks, &block); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pReader->pSchemaWrapper); + taosArrayPush(schemas, &pSW); + return 0; + } + return -1; +} void tqReaderSetColIdList(STqReader* pReader, SArray* pColIdList) { pReader->pColIdList = pColIdList; } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 5907be576a67441f12c4045ba33243b751451fd4..b9969409d17f99b1d830e97becb2b1a2c797635a 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -72,6 +72,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } +#if 0 SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, SBatchDeleteReq* pDeleteReq) { @@ -252,8 +253,6 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem int32_t rows = pDataBlock->info.rows; - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - int32_t dataLen = 0; int32_t schemaLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -299,6 +298,201 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem return ret; } +#endif + +int32_t tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, + SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, + SBatchDeleteReq* pDeleteReq, void** ppData, int32_t* pLen) { + void* pBuf = NULL; + int32_t len = 0; + SSubmitReq2* pReq = NULL; + SArray* tagArray = NULL; + SArray* createTbArray = NULL; + SArray* pVals = NULL; + + int32_t sz = taosArrayGetSize(pBlocks); + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + // create table req + if (createTb) { + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + SVCreateTbReq* pCreateTbReq = NULL; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + taosArrayPush(createTbArray, &pCreateTbReq); + continue; + } + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // don't move to the end of loop as to destroy in the end of func when error occur + taosArrayPush(createTbArray, &pCreateTbReq); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + pCreateTbReq->name = strdup(pDataBlock->info.parTbName); + } else { + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + } + } + + // SSubmitTbData req + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } + + int32_t rows = pDataBlock->info.rows; + + SSubmitTbData* pTbData = (SSubmitTbData*)taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (!pTbData) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pTbData->aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + taosMemoryFree(pTbData); + goto _end; + } + pTbData->suid = suid; + pTbData->uid = 0; // uid is assigned by vnode + pTbData->sver = pTSchema->version; + + if (createTb) { + pTbData->pCreateTbReq = taosArrayGetP(createTbArray, i); + if (pTbData->pCreateTbReq) pTbData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } + + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(pTbData->aRowP); + taosMemoryFree(pTbData); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(pTbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(pTbData->aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, pTbData); + } + + // encode + tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + if (TSDB_CODE_SUCCESS == terrno) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = htonl(TD_VID(pVnode)); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + } + tEncoderClear(&encoder); + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + + if (terrno != 0) { + rpcFreeCont(pBuf); + taosArrayDestroy(pDeleteReq->deleteReqs); + return TSDB_CODE_FAILED; + } + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + return TSDB_CODE_SUCCESS; +} void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; @@ -493,7 +687,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); + tqDebug("tq sink pipe1, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -522,7 +716,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } else { void* colData = colDataGetData(pColData, j); if (k == 0) { - tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + tqDebug("tq sink pipe1, row %d ts %" PRId64, j, *(int64_t*)colData); } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } @@ -551,6 +745,253 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d taosArrayDestroy(tagArray); } +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ + + int32_t blockSz = taosArrayGetSize(pBlocks); + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + + void* pBuf = NULL; + SSubmitReq2* pReq = NULL; + SArray* tagArray = NULL; + SArray* pVals = NULL; + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + continue; + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else { + SSubmitTbData tbData = {0}; + int32_t rows = pDataBlock->info.rows; + tqDebug("tq sink pipe2, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + if (pDataBlock->info.parTbName[0]) { + ctbName = strdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq* pCreateTbReq = NULL; + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); + } + + // rows + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (k == 0) { + void* colData = colDataGetData(pColData, j); + tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + } + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* colData = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + + // encode + int32_t len; + int32_t code; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SMsgHead*)pBuf)->vgId = TD_VID(pVnode); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; + } + tEncoderClear(&encoder); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pBuf, + .contLen = len, + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + // TODO: change + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); +} + #if 0 void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pRes = (const SArray*)data; diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index b68763867efcf9a8e7c13ec43bfb827df75b5959..d811d943ed9c45f30e64ac717a93e88587ed3fcc 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { - tdbAbort(pWriter->pTq->pMetaDB, &pWriter->txn); + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); } else { - code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 08d5931bc3250895d2f2eea5b9c164e54ae4c127..b1f00bdf7446789fa3c572ff366bd20319db10ec 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); ASSERT(0); } else { - code = tdbCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaDB, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 31e44a5b6dcd170d801b3aa0481b6dd7ea851f0b..305378bc932f0484c71a9d1c91935a580189488e 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -129,7 +129,7 @@ struct STqSnapWriter { STQ* pTq; int64_t sver; int64_t ever; - TXN txn; + TXN* txn; }; int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) { @@ -146,8 +146,10 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbTxnOpen(&pWriter->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { - ASSERT(0); + if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + code = -1; + taosMemoryFree(pWriter); + goto _err; } *ppWriter = pWriter; @@ -165,11 +167,12 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { + tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn); ASSERT(0); } else { - code = tdbCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaStore, &pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 33351c710b8749bcda2f721281c2677a55e1cae0..7375148d259dbc324e9cb2af80ccd30b575e2fbb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -91,12 +91,10 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -124,12 +122,10 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -208,6 +204,44 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TS int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -228,23 +262,23 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TS invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + } else { // new inserting key is greater than cached, update cached entry + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -253,65 +287,10 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TS taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - /* - cacheRow = (STSRow *)taosLRUCacheValue(pCache, h); - if (row->ts >= cacheRow->ts) { - if (row->ts == cacheRow->ts) { - STSRow *mergedRow = NULL; - SRowMerger merger = {0}; - STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - - tsdbRowMergerInit(&merger, &tsdbRowFromTSRow(0, cacheRow), pTSchema); - - tsdbRowMerge(&merger, &tsdbRowFromTSRow(1, row)); - - tsdbRowMergerGetRow(&merger, &mergedRow); - tsdbRowMergerClear(&merger); - - taosMemoryFreeClear(pTSchema); - - row = mergedRow; - dup = false; - } - - if (TD_ROW_LEN(row) <= TD_ROW_LEN(cacheRow)) { - tdRowCpy(cacheRow, row); - if (!dup) { - taosMemoryFree(row); - } - - taosLRUCacheRelease(pCache, h, false); - } else { - taosLRUCacheRelease(pCache, h, true); - // tsdbCacheDeleteLastrow(pCache, uid, TSKEY_MAX); - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; - } - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, - TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } - // tsdbCacheInsertLastrow(pCache, uid, row, dup); - } - }*/ - } /*else { - if (dup) { - cacheRow = tdRowDup(row); - } else { - cacheRow = row; - } - - _taos_lru_deleter_t deleter = deleteTableCacheLastrow; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, cacheRow, TD_ROW_LEN(cacheRow), deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); } - }*/ + } return code; } @@ -334,6 +313,44 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; @@ -349,28 +366,28 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb SColVal colVal = {0}; tsdbRowGetColVal(row, pTSchema, iCol, &colVal); - if (!COL_VAL_IS_VALUE(&colVal)) { + if (COL_VAL_IS_VALUE(&colVal)) { if (keyTs == tTsVal1->ts && COL_VAL_IS_VALUE(tColVal)) { invalidate = true; break; - } - } else { - SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; - if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { - SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); - taosMemoryFree(pLastCol->colVal.value.pData); - - lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); - if (lastCol.colVal.value.pData == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = TSDB_CODE_OUT_OF_MEMORY; - goto _invalidate; + } else { + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); } - memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); - } - taosArraySet(pLast, iCol, &lastCol); + taosArraySet(pLast, iCol, &lastCol); + } } } } @@ -379,9 +396,9 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb taosMemoryFreeClear(pTSchema); taosLRUCacheRelease(pCache, h, invalidate); - - // clear last cache anyway, lazy load when get last lookup - // taosLRUCacheRelease(pCache, h, true); + if (invalidate) { + taosLRUCacheErase(pCache, key, keyLen); + } } return code; @@ -740,7 +757,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); + */ tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); @@ -1007,12 +1025,17 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs SArray *pDelIdxArray = taosArrayInit(32, sizeof(SDelIdx)); code = tsdbReadDelIdx(pDelFReader, pDelIdxArray); - if (code) goto _err; + if (code) { + taosArrayDestroy(pDelIdxArray); + tsdbDelFReaderClose(&pDelFReader); + goto _err; + } SDelIdx *delIdx = taosArraySearch(pDelIdxArray, &(SDelIdx){.suid = suid, .uid = uid}, tCmprDelIdx, TD_EQ); code = getTableDelSkyline(pMem, pIMem, pDelFReader, delIdx, pIter->pSkyline); if (code) { + taosArrayDestroy(pDelIdxArray); tsdbDelFReaderClose(&pDelFReader); goto _err; } @@ -1269,12 +1292,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // build the result ts row here *dup = false; - if (taosArrayGetSize(pColArray) != nCol) { - *ppColArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppColArray = pColArray; - } + // if (taosArrayGetSize(pColArray) != nCol) { + //*ppColArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppColArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1387,12 +1410,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } } while (setNoneCol); - if (taosArrayGetSize(pColArray) <= 0) { - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppLastArray = pColArray; - } + // if (taosArrayGetSize(pColArray) <= 0) { + //*ppLastArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppLastArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1423,8 +1446,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pArray == NULL*/) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1442,20 +1465,14 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; return code; } - // int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pTSchema) { // int32_t code = 0; // int16_t nCol = taosArrayGetSize(pLastArray); @@ -1496,8 +1513,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pLastArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pLastArray == NULL*/) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1511,13 +1528,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index b87e5d5503914772b42ca038c434552310fca324..76e5897e53de3ce6135701333e8d89682bb1cbdc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -253,6 +253,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } { for (int32_t k = 0; k < pr->numOfCols; ++k) { @@ -322,6 +326,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } saveOneRow(pRow, pResBlock, pr, slotIds, pRes); // TODO reset the pRes diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 659c0e1f08a03937e6f56c9147bb9888db1ad6f2..92451dcf5805d67fe053513f958470a5b118b4a2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -53,6 +53,7 @@ typedef struct { // -------------- TSKEY nextKey; // reset by each table commit int32_t commitFid; + int32_t expLevel; TSKEY minKey; TSKEY maxKey; // commit file data @@ -93,7 +94,7 @@ typedef struct { SArray *aDelData; // SArray } SCommitter; -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter); +static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo); static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); static int32_t tsdbCommitCache(SCommitter *pCommitter); @@ -150,18 +151,28 @@ _exit: return code; } -int32_t tsdbCommit(STsdb *pTsdb) { +int32_t tsdbPrepareCommit(STsdb *pTsdb) { + taosThreadRwlockWrlock(&pTsdb->rwLock); + ASSERT(pTsdb->imem == NULL); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + taosThreadRwlockUnlock(&pTsdb->rwLock); + + return 0; +} + +int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { if (!pTsdb) return 0; int32_t code = 0; int32_t lino = 0; SCommitter commith; - SMemTable *pMemTable = pTsdb->mem; + SMemTable *pMemTable = pTsdb->imem; // check if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->mem = NULL; + pTsdb->imem = NULL; taosThreadRwlockUnlock(&pTsdb->rwLock); tsdbUnrefMemTable(pMemTable); @@ -169,7 +180,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { } // start commit - code = tsdbStartCommit(pTsdb, &commith); + code = tsdbStartCommit(pTsdb, &commith, pInfo); TSDB_CHECK_CODE(code, lino, _exit); // commit impl @@ -493,6 +504,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // memory pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); + pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec()); tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, &pCommitter->maxKey); #if 0 @@ -546,7 +558,10 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } } else { SDiskID did = {0}; - tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); + if (tfsAllocDisk(pTsdb->pVnode->pTfs, pCommitter->expLevel, &did) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); wSet.diskId = did; wSet.nSttF = 1; @@ -806,26 +821,21 @@ _exit: } // ---------------------------------------------------------------------------- -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { +static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo) { int32_t code = 0; int32_t lino = 0; memset(pCommitter, 0, sizeof(*pCommitter)); - ASSERT(pTsdb->mem && pTsdb->imem == NULL && "last tsdb commit incomplete"); - - taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); + ASSERT(pTsdb->imem && "last tsdb commit incomplete"); pCommitter->pTsdb = pTsdb; - pCommitter->commitID = pTsdb->pVnode->state.commitID; + pCommitter->commitID = pInfo->info.state.commitID; pCommitter->minutes = pTsdb->keepCfg.days; pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; - pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger; + pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows; + pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows; + pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression; + pCommitter->sttTrigger = pInfo->info.config.sttTrigger; pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 72d9c4f69ef19908a84e27d9ecfe03b1aa776337..7dc839773f11b6bb823375f35281977e94b74f8b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -962,6 +962,7 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } } + pDFileSet->diskId = pSet->diskId; goto _exit; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index baac0fc8f7fa1b34017c726a50b8d5f8ff83b129..007ae871a0e20d84d729f2d8123c0e7500591423 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -569,9 +569,9 @@ static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - ASSERT(aColData[0].cid = PRIMARYKEY_TIMESTAMP_COL_ID); - ASSERT(aColData[0].type = TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(aColData[0].flag = HAS_VALUE); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].flag == HAS_VALUE); // copy and construct block data SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); @@ -695,7 +695,7 @@ static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, } while (iRow < nRow) { - tRow.pTSRow = aRow[iRow++]; + tRow.pTSRow = aRow[iRow]; key.ts = tRow.pTSRow->ts; if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 48575b108f1ed54453ae2a42a0fc3c06a18467ca..eebfde49db1985c79a9fc143b2a256076131a284 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -121,7 +121,7 @@ static SBlockData *loadLastBlock(SLDataIter *pIter, const char *idStr) { return &pInfo->blockData[1]; } - if (pIter->pSttBlk == NULL) { + if (pIter->pSttBlk == NULL || pInfo->pSchema == NULL) { return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 3a08093d47f39ef915149b82de890625189e7874..cddc8a213daa2a8b7b46882bb708aa29876ae08f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -82,13 +82,13 @@ typedef struct SIOCostSummary { } SIOCostSummary; typedef struct SBlockLoadSuppInfo { - SArray* pColAgg; - SColumnDataAgg tsColAgg; - SColumnDataAgg** plist; - int16_t* colIds; // column ids for loading file block data - int32_t numOfCols; - char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. - bool smaValid; // the sma on all queried columns are activated + SArray* pColAgg; + SColumnDataAgg tsColAgg; + int16_t* colId; + int16_t* slotId; + int32_t numOfCols; + char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. + bool smaValid; // the sma on all queried columns are activated } SBlockLoadSuppInfo; typedef struct SLastBlockReader { @@ -158,6 +158,7 @@ struct STsdbReader { STsdb* pTsdb; uint64_t suid; int16_t order; + bool freeBlock; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -167,9 +168,11 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times - SDataFReader* pFileReader; + STSchema* pSchema; // the newest version schema + STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; SVersionRange verRange; SBlockInfoBuf blockInfoBuf; int32_t step; @@ -194,8 +197,8 @@ static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange); -static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pTSRow, - STsdbReader* pReader, bool* freeTSRow); +static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, + TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, @@ -211,28 +214,30 @@ static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } -static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SSDataBlock* pBlock) { - size_t numOfCols = blockDataGetNumOfCols(pBlock); - +static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, + int32_t numOfCols) { pSupInfo->smaValid = true; pSupInfo->numOfCols = numOfCols; - pSupInfo->colIds = taosMemoryMalloc(numOfCols * sizeof(int16_t)); - pSupInfo->buildBuf = taosMemoryCalloc(numOfCols, POINTER_BYTES); - if (pSupInfo->buildBuf == NULL || pSupInfo->colIds == NULL) { - taosMemoryFree(pSupInfo->colIds); - taosMemoryFree(pSupInfo->buildBuf); + pSupInfo->colId = taosMemoryMalloc(numOfCols * (sizeof(int16_t) * 2 + POINTER_BYTES)); + if (pSupInfo->colId == NULL) { + taosMemoryFree(pSupInfo->colId); return TSDB_CODE_OUT_OF_MEMORY; } + pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols)); + pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); - pSupInfo->colIds[i] = pCol->info.colId; + pSupInfo->colId[i] = pCols[i].colId; + pSupInfo->slotId[i] = pSlotIdList[i]; - if (IS_VAR_DATA_TYPE(pCol->info.type)) { - pSupInfo->buildBuf[i] = taosMemoryMalloc(pCol->info.bytes); + if (IS_VAR_DATA_TYPE(pCols[i].type)) { + pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes); + } else { + pSupInfo->buildBuf[i] = NULL; } } @@ -244,7 +249,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) { STColumn* pTCol = &pSchema->columns[i]; - if (pTCol->colId == pSupInfo->colIds[j]) { + if (pTCol->colId == pSupInfo->colId[j]) { if (!IS_BSMA_ON(pTCol)) { pSupInfo->smaValid = false; return; @@ -252,7 +257,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) i += 1; j += 1; - } else if (pTCol->colId < pSupInfo->colIds[j]) { + } else if (pTCol->colId < pSupInfo->colId[j]) { // do nothing i += 1; } else { @@ -455,7 +460,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb if (pLReader->pInfo == NULL) { // here we ignore the first column, which is always be the primary timestamp column pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tCreateLastBlockLoadInfo(pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -567,7 +572,7 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) } static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, - const char* idstr) { + SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; int8_t level = 0; STsdbReader* pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader)); @@ -586,6 +591,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->suid = pCond->suid; pReader->order = pCond->order; pReader->capacity = capacity; + pReader->pResBlock = pResBlock; pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; @@ -593,13 +599,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket ASSERT(pCond->numOfCols > 0); + if (pReader->pResBlock == NULL) { + pReader->freeBlock = true; + pReader->pResBlock = createResBlock(pCond, pReader->capacity); + if (pReader->pResBlock == NULL) { + code = terrno; + goto _end; + } + } + + // todo refactor. limitOutputBufferSize(pCond, &pReader->capacity); // allocate buffer in order to load data blocks from file SBlockLoadSuppInfo* pSup = &pReader->suppInfo; pSup->pColAgg = taosArrayInit(pCond->numOfCols, sizeof(SColumnDataAgg)); - pSup->plist = taosMemoryCalloc(pCond->numOfCols, POINTER_BYTES); - if (pSup->pColAgg == NULL || pSup->plist == NULL) { + if (pSup->pColAgg == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } @@ -612,13 +627,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->pResBlock = createResBlock(pCond, pReader->capacity); - if (pReader->pResBlock == NULL) { - code = terrno; - goto _end; - } - - setColumnIdSlotList(&pReader->suppInfo, pReader->pResBlock); + setColumnIdSlotList(&pReader->suppInfo, pCond->colList, pCond->pSlotList, pCond->numOfCols); *ppReader = pReader; return code; @@ -1043,17 +1052,16 @@ static void copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo } static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { - SReaderStatus* pStatus = &pReader->status; - SDataBlockIter* pBlockIter = &pStatus->blockIter; + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pStatus->fileBlockData; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; - int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); - - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int32_t numOfOutputCols = pSupInfo->numOfCols; SColVal cv = {0}; int64_t st = taosGetTimestampUs(); @@ -1089,8 +1097,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t i = 0; int32_t rowIndex = 0; - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc); i += 1; } @@ -1099,12 +1107,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t num = pBlockData->nColData; while (i < numOfOutputCols && colIndex < num) { rowIndex = 0; - pColData = taosArrayGet(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex); - if (pData->cid < pColData->info.colId) { + if (pData->cid < pSupInfo->colId[i]) { colIndex += 1; - } else if (pData->cid == pColData->info.colId) { + } else if (pData->cid == pSupInfo->colId[i]) { + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) { colDataAppendNNULL(pColData, 0, dumpedRows); } else { @@ -1121,6 +1130,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn colIndex += 1; i += 1; } else { // the specified column does not exist in file block, fill with null data + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } @@ -1128,11 +1138,12 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // fill the mis-matched columns with null value while (i < numOfOutputCols) { - pColData = taosArrayGet(pResBlock->pDataBlock, i); + pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNNULL(pColData, 0, dumpedRows); i += 1; } + pResBlock->info.dataLoad = 1; pResBlock->info.rows = dumpedRows; pDumpInfo->rowIndex += step * dumpedRows; @@ -1147,6 +1158,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn setBlockAllDumped(pDumpInfo, ts, pReader->order); } + pBlockScanInfo->lastKey = pDumpInfo->lastKey; + double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -1166,7 +1179,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); TABLEID tid = {.suid = pReader->suid, .uid = uid}; int32_t code = - tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols - 1); + tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colId[1], pReader->suppInfo.numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1304,7 +1317,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte char* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num); if (buf == NULL) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; @@ -1347,7 +1360,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanupBlockOrderSupporter(&sup); - return TSDB_CODE_TDB_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t numOfTotal = 0; @@ -1625,7 +1638,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* int64_t st = taosGetTimestampUs(); int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); - blockDataUpdateTsWindow(pBlock, 0); + blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); pBlock->info.id.uid = pBlockScanInfo->uid; setComposedBlockFlag(pReader, true); @@ -2458,7 +2471,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { bool hasBlockData = false; { - while (pBlockData->nRow > 0) { // find the first qualified row in data block + while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2468,8 +2481,42 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { - setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); - break; + SBlockIndex bIndex = {0}; + int32_t nextIndex = -1; + bool hasNeighbor = false; + if (pBlockInfo != NULL) { + hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pBlockScanInfo, &nextIndex, pReader->order, &bIndex); + } + + if (!hasNeighbor) { // do nothing + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + break; + } + + if (overlapWithNeighborBlock(pBlock, &bIndex, pReader->order)) { // load next block + SReaderStatus* pStatus = &pReader->status; + SDataBlockIter* pBlockIter = &pStatus->blockIter; + + // 1. find the next neighbor block in the scan block list + SFileDataBlockInfo fb = {.uid = pBlockInfo->uid, .tbBlockIdx = nextIndex}; + int32_t neighborIndex = findFileBlockInfoIndex(pBlockIter, &fb); + + // 2. remove it from the scan block list + setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); + + // 3. load the neighbor block, and set it to be the currently accessed file data block + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid); + if (code != TSDB_CODE_SUCCESS) { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + break; + } + + // 4. check the data values + initBlockDumpInfo(pReader, pBlockIter); + } else { + setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + break; + } } } } @@ -2497,7 +2544,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { _end: pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; - blockDataUpdateTsWindow(pResBlock, 0); + pResBlock->info.dataLoad = 1; + blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]); setComposedBlockFlag(pReader, true); double el = (taosGetTimestampUs() - st) / 1000.0; @@ -2524,42 +2572,18 @@ int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* } int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; - SArray* pDelData = taosArrayInit(4, sizeof(SDelData)); + ASSERT(pReader->pReadSnap != NULL); SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; - if (pDelFile) { - SDelFReader* pDelFReader = NULL; - code = tsdbDelFReaderOpen(&pDelFReader, pDelFile, pTsdb); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - SArray* aDelIdx = taosArrayInit(4, sizeof(SDelIdx)); - if (aDelIdx == NULL) { - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - - // TODO: opt the perf of read del index - code = tsdbReadDelIdx(pDelFReader, aDelIdx); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - goto _err; - } - + if (pDelFile && taosArrayGetSize(pReader->pDelIdx) > 0) { SDelIdx idx = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - SDelIdx* pIdx = taosArraySearch(aDelIdx, &idx, tCmprDelIdx, TD_EQ); + SDelIdx* pIdx = taosArraySearch(pReader->pDelIdx, &idx, tCmprDelIdx, TD_EQ); if (pIdx != NULL) { - code = tsdbReadDelData(pDelFReader, pIdx, pDelData); + code = tsdbReadDelData(pReader->pDelFReader, pIdx, pDelData); } - taosArrayDestroy(aDelIdx); - tsdbDelFReaderClose(&pDelFReader); - if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -2655,6 +2679,29 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { } taosArrayDestroy(pIndexList); + + if (pReader->pReadSnap != NULL) { + SDelFile* pDelFile = pReader->pReadSnap->fs.pDelFile; + if (pReader->pDelFReader == NULL && pDelFile != NULL) { + int32_t code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pReader->pTsdb); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + pReader->pDelIdx = taosArrayInit(4, sizeof(SDelIdx)); + if (pReader->pDelIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + code = tsdbReadDelIdx(pReader->pDelFReader, pReader->pDelIdx); + if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pReader->pDelIdx); + return code; + } + } + } + return TSDB_CODE_SUCCESS; } @@ -2749,7 +2796,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { @@ -2879,7 +2927,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } // set the correct start position in case of the first/last file block, according to the query time window -static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { +void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SDataBlk* pBlock = getCurrentBlock(pBlockIter); SReaderStatus* pStatus = &pReader->status; @@ -2980,7 +3028,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } else { if (pReader->status.pCurrentFileset->nSttF > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); resetDataBlockIterator(pBlockIter, pReader->order); goto _begin; } else { @@ -3088,7 +3141,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 return false; } else if (pKey->ts == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version); + return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); @@ -3232,12 +3285,16 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - tsdbRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMerge(pMerger, pRow); + } } return TSDB_CODE_SUCCESS; @@ -3333,6 +3390,11 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3386,46 +3448,55 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } } -#if 0 - // todo handle the data block merge SRowMerger merge = {0}; - - // get the correct schema for data in memory terrno = 0; - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + int32_t code = 0; - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - int32_t code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pSchema == NULL) { + pReader->pSchema = pTSchema; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } + + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + tsdbRowMerge(&merge, pNextRow); + } code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tsdbRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } + pResRow->type = current.type; tsdbRowMergerClear(&merge); *freeTSRow = true; -#endif return TSDB_CODE_SUCCESS; } @@ -3543,33 +3614,37 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { - int32_t numOfRows = pBlock->info.rows; - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); SColVal colVal = {0}; int32_t i = 0, j = 0; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (pColInfoData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppend(pColInfoData, numOfRows, (const char*)&pTSRow->ts, false); + if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts; i += 1; } - while (i < numOfCols && j < pSchema->numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - col_id_t colId = pColInfoData->info.colId; + while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) { + col_id_t colId = pSupInfo->colId[i]; if (colId == pSchema->columns[j].colId) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + tRowGet(pTSRow, pSchema, j, &colVal); - doCopyColVal(pColInfoData, numOfRows, i, &colVal, pSupInfo); + doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; } else if (colId < pSchema->columns[j].colId) { - colDataAppendNULL(pColInfoData, numOfRows); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } else if (colId > pSchema->columns[j].colId) { j += 1; @@ -3577,12 +3652,13 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT } // set null value since current column does not exist in the "pSchema" - while (i < numOfCols) { - pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - colDataAppendNULL(pColInfoData, numOfRows); + while (i < pSupInfo->numOfCols) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); + colDataAppendNULL(pColInfoData, outputRowIndex); i += 1; } + pBlock->info.dataLoad = 1; pBlock->info.rows += 1; pScanInfo->lastKey = pTSRow->ts; return TSDB_CODE_SUCCESS; @@ -3594,27 +3670,25 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t outputRowIndex = pResBlock->info.rows; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); - if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - colDataAppendInt64(pColData, outputRowIndex, &pBlockData->aTSKEY[rowIndex]); + if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); + ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; } SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; - int32_t numOfOutputCols = pResBlock->pDataBlock->size; + int32_t numOfOutputCols = pSupInfo->numOfCols; while (i < numOfOutputCols && j < numOfInputCols) { - SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i); - SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); - - if (pData->cid < pCol->info.colId) { + SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pSupInfo->colId[i]) { j += 1; continue; } - if (pData->cid == pCol->info.colId) { + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]); + if (pData->cid == pSupInfo->colId[i]) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; @@ -3627,11 +3701,12 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } while (i < numOfOutputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); colDataAppendNULL(pCol, outputRowIndex); i += 1; } + pResBlock->info.dataLoad = 1; pResBlock->info.rows += 1; return TSDB_CODE_SUCCESS; } @@ -3641,9 +3716,9 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e SSDataBlock* pBlock = pReader->pResBlock; do { -// SRow* pTSRow = NULL; + // SRow* pTSRow = NULL; TSDBROW row = {.type = -1}; - bool freeTSRow = false; + bool freeTSRow = false; tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); if (row.type == -1) { break; @@ -3731,14 +3806,21 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { STimeWindow window = pCond->twindows; if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { pCond->twindows.skey += 1; pCond->twindows.ekey -= 1; } - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, 4096, idstr); + int32_t capacity = 0; + if (pResBlock == NULL) { + capacity = 4096; + } else { + capacity = pResBlock->info.capacity; + } + + int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3764,7 +3846,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } // here we only need one more row, so the capacity is set to be ONE. - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[0], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3778,13 +3860,15 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } pCond->order = order; - code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, idstr); + code = tsdbReaderCreate(pVnode, pCond, &pReader->innerReader[1], 1, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; } } // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. + // no valid error code set in metaGetTbTSchema, so let's set the error code here. + // we should proceed in case of tmq processing. if (pCond->suid != 0) { pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, -1, 1); if (pReader->pSchema == NULL) { @@ -3808,7 +3892,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL tsdbReaderClose(p); *ppReader = NULL; - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -3852,6 +3936,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); + tsdbReaderClose(pReader); return code; } @@ -3883,19 +3968,19 @@ void tsdbReaderClose(STsdbReader* pReader) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - taosMemoryFreeClear(pSupInfo->plist); - taosMemoryFree(pSupInfo->colIds); - taosArrayDestroy(pSupInfo->pColAgg); - for (int32_t i = 0; i < blockDataGetNumOfCols(pReader->pResBlock); ++i) { + for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) { if (pSupInfo->buildBuf[i] != NULL) { taosMemoryFreeClear(pSupInfo->buildBuf[i]); } } - taosMemoryFree(pSupInfo->buildBuf); - tBlockDataDestroy(&pReader->status.fileBlockData); + if (pReader->freeBlock) { + pReader->pResBlock = blockDataDestroy(pReader->pResBlock); + } + taosMemoryFree(pSupInfo->colId); + tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -3904,12 +3989,19 @@ void tsdbReaderClose(STsdbReader* pReader) { clearBlockScanInfoBuf(&pReader->blockInfoBuf); } - blockDataDestroy(pReader->pResBlock); - if (pReader->pFileReader != NULL) { tsdbDataFReaderClose(&pReader->pFileReader); } + if (pReader->pDelFReader != NULL) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + + if (pReader->pDelIdx != NULL) { + taosArrayDestroy(pReader->pDelIdx); + pReader->pDelIdx = NULL; + } + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); @@ -3951,6 +4043,9 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { blockDataCleanup(pBlock); SReaderStatus* pStatus = &pReader->status; + if (taosHashGetSize(pStatus->pTableMap) == 0) { + return false; + } if (pStatus->loadFromFile) { int32_t code = buildBlockFromFiles(pReader); @@ -3968,8 +4063,6 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { buildBlockFromBufferSequentially(pReader); return pBlock->info.rows > 0; } - - return false; } bool tsdbNextDataBlock(STsdbReader* pReader) { @@ -4020,16 +4113,6 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } -bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { - STableBlockScanInfo* pBlockScanInfo = - *(STableBlockScanInfo**)taosHashGet(pReader->status.pTableMap, &uid, sizeof(uid)); - if (pBlockScanInfo == NULL) { // no data block for the table of given uid - return false; - } - - return true; -} - static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { ASSERT(pReader != NULL); *rows = pReader->pResBlock->info.rows; @@ -4051,28 +4134,51 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64 } } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockStatis, bool* allHave) { +static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + // do fill all null column value SMA info + int32_t i = 0, j = 0; + int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); + taosArrayInsert(pSup->pColAgg, 0, pTsAgg); + + while (j < numOfCols && i < size) { + SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); + if (pAgg->colId == pSup->colId[j]) { + i += 1; + j += 1; + } else if (pAgg->colId < pSup->colId[j]) { + i += 1; + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) { + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; + taosArrayInsert(pSup->pColAgg, i, &nullColAgg); + } + j += 1; + } + } +} + +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { int32_t code = 0; + SColumnDataAgg ***pBlockSMA = &pDataBlock->pBlockAgg; *allHave = false; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } // there is no statistics data for composed block if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - - SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - // int64_t stime = taosGetTimestampUs(); - SBlockLoadSuppInfo* pSup = &pReader->suppInfo; + ASSERT(pReader->pResBlock->info.id.uid == pFBlock->uid); + + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { @@ -4081,7 +4187,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS return code; } } else { - *pBlockStatis = NULL; + *pBlockSMA = NULL; return TSDB_CODE_SUCCESS; } @@ -4094,80 +4200,63 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; pTsAgg->min = pReader->pResBlock->info.window.skey; pTsAgg->max = pReader->pResBlock->info.window.ekey; - pSup->plist[0] = pTsAgg; // update the number of NULL data rows - size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); + size_t numOfCols = pSup->numOfCols; int32_t i = 0, j = 0; size_t size = taosArrayGetSize(pSup->pColAgg); -#if 0 - while (j < numOfCols && i < size) { - SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { - pSup->plist[j] = pAgg; - } else { - *allHave = false; - break; - } - i += 1; - j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { - i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - j += 1; - } + + // ensure capacity + if(pDataBlock->pDataBlock) { + size_t colsNum = taosArrayGetSize(pDataBlock->pDataBlock); + taosArrayEnsureCap(pSup->pColAgg, colsNum); } -#else - // fill the all null data column - SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg)); + SSDataBlock* pResBlock = pReader->pResBlock; + if (pResBlock->pBlockAgg == NULL) { + size_t num = taosArrayGetSize(pResBlock->pDataBlock); + pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); + } + + // do fill all null column value SMA info + doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + i = 0, j = 0; while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); - if (pAgg->colId == pSup->colIds[j]) { - taosArrayPush(pNewAggList, pAgg); + if (pAgg->colId == pSup->colId[j]) { + pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; i += 1; j += 1; - } else if (pAgg->colId < pSup->colIds[j]) { + } else if (pAgg->colId < pSup->colId[j]) { i += 1; - } else if (pSup->colIds[j] < pAgg->colId) { - if (pSup->colIds[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { - taosArrayPush(pNewAggList, &pSup->tsColAgg); + } else if (pSup->colId[j] < pAgg->colId) { + if (pSup->colId[j] == PRIMARYKEY_TIMESTAMP_COL_ID) { + pResBlock->pBlockAgg[pSup->slotId[j]] = &pSup->tsColAgg; } else { // all date in this block are null - SColumnDataAgg nullColAgg = {.colId = pSup->colIds[j], .numOfNull = pBlock->nRow}; - taosArrayPush(pNewAggList, &nullColAgg); + SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = pBlock->nRow}; + taosArrayPush(pSup->pColAgg, &nullColAgg); + + pResBlock->pBlockAgg[pSup->slotId[j]] = taosArrayGetLast(pSup->pColAgg); } j += 1; } } - taosArrayClear(pSup->pColAgg); - taosArrayAddAll(pSup->pColAgg, pNewAggList); - - size_t num = taosArrayGetSize(pSup->pColAgg); - for (int32_t k = 0; k < num; ++k) { - pSup->plist[k] = taosArrayGet(pSup->pColAgg, k); - } - - taosArrayDestroy(pNewAggList); - -#endif - + *pBlockSMA = pResBlock->pBlockAgg; pReader->cost.smaDataLoad += 1; - *pBlockStatis = pSup->plist; tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } -static SArray* doRetrieveDataBlock(STsdbReader* pReader) { +static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; if (pStatus->composedDataBlock) { - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); @@ -4188,10 +4277,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - return pReader->pResBlock->pDataBlock; + return pReader->pResBlock; } -SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { +SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { return doRetrieveDataBlock(pReader->innerReader[0]); @@ -4218,7 +4307,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { // allocate buffer in order to load data blocks from file memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg)); - memset(pReader->suppInfo.plist, 0, POINTER_BYTES); pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 294a4bd3e4c404ddb33a4e3fe7cd0c5fe2203a87..c7bce6182a380bd278a8e272eae22773350a630c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -750,7 +750,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // head tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -771,7 +771,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // data tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -781,7 +781,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pDataF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -792,7 +792,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { // sma tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -814,7 +814,7 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + pOutFD = taosCreateFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pOutFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -1477,9 +1477,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { } taosMemoryFree(pReader); } - *ppReader = NULL; -_exit: + *ppReader = NULL; return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 048d3cc22b5d2b92bf12919fe89b529f1589576d..a7a06ee946b7a143ff9ce92637262a590cde204d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -155,7 +155,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { if (rowVer >= pReader->sver && rowVer <= pReader->ever) { pIter->rInfo.suid = pIter->bData.suid; - pIter->rInfo.uid = pIter->bData.uid; + pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); goto _add_iter; } @@ -179,16 +179,14 @@ _err: return code; } -static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; } - static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { int32_t code = 0; if (pReader->pIter) { - SFDataIter* pIter = pReader->pIter; - + SFDataIter* pIter = NULL; while (true) { _find_row: + pIter = pReader->pIter; for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; @@ -224,6 +222,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { } pReader->pIter = NULL; + break; } else if (pIter->type == SNAP_STT_FILE_ITER) { for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); @@ -238,6 +237,7 @@ static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { } pReader->pIter = NULL; + break; } else { ASSERT(0); } @@ -269,6 +269,20 @@ _err: return code; } +static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { + if (pReader->pIter) { + return &pReader->pIter->rInfo; + } else { + tsdbSnapNextRow(pReader); + + if (pReader->pIter) { + return &pReader->pIter->rInfo; + } else { + return NULL; + } + } +} + static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; @@ -1356,33 +1370,40 @@ _exit: taosMemoryFree(pWriter); } } else { - tsdbDebug("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); *ppWriter = pWriter; } return code; } +int32_t tsdbSnapWriterPrepareClose(STsdbSnapWriter* pWriter) { + int32_t code = 0; + if (pWriter->dWriter.pWriter) { + code = tsdbSnapWriteCloseFile(pWriter); + if (code) goto _exit; + } + + code = tsdbSnapWriteDelEnd(pWriter); + if (code) goto _exit; + + code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs); + if (code) goto _exit; + +_exit: + if (code) { + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); + } + return code; +} + int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; STsdbSnapWriter* pWriter = *ppWriter; STsdb* pTsdb = pWriter->pTsdb; if (rollback) { - ASSERT(0); - // code = tsdbFSRollback(pWriter->pTsdb->pFS); - // if (code) goto _err; + tsdbRollbackCommit(pWriter->pTsdb); } else { - if (pWriter->dWriter.pWriter) { - code = tsdbSnapWriteCloseFile(pWriter); - if (code) goto _err; - } - - code = tsdbSnapWriteDelEnd(pWriter); - if (code) goto _err; - - code = tsdbFSPrepareCommit(pWriter->pTsdb, &pWriter->fs); - if (code) goto _err; - // lock taosThreadRwlockWrlock(&pTsdb->rwLock); @@ -1421,7 +1442,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); } - tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + tsdbInfo("vgId:%d %s done", TD_VID(pWriter->pTsdb->pVnode), __func__); taosMemoryFree(pWriter); *ppWriter = NULL; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 36ee85df6b89e2a4e989d8b32200f81ae7381d64..1c92e922b0004c6ff9cc94b5e6fafa80b638cc0f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -107,7 +107,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem SArray *pArray = taosArrayInit(pMapData->nItem, itemSize); if (pArray == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -637,6 +637,7 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { } } else { ASSERT(0); + return NULL; // suppress error report by compiler } } @@ -1105,8 +1106,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD int32_t ridx = pBlockData->nColData - 1; while (lidx <= ridx) { - int32_t midx = lidx + (ridx - lidx) * (cid - pBlockData->aColData[lidx].cid) / - (pBlockData->aColData[ridx].cid - pBlockData->aColData[lidx].cid); + int32_t midx = (lidx + ridx) >> 1; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 0118d658c29e2356e6507e38c9a276a8faca83b5..7329bd65fbf804dbb16af721e975f8f772512e97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -26,14 +26,17 @@ static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 921 // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - int32_t affectedrows = 0; - int32_t numOfRows = 0; +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t arrSize = 0; + int32_t affectedrows = 0; + int32_t numOfRows = 0; ASSERT(pTsdb->mem != NULL); + if (pMsg) { + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); + } + // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { @@ -43,22 +46,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - SSubmitBlkRsp r = {0}; - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; -#if 0 - if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { + for (int32_t i = 0; i < arrSize; ++i) { + if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { return -1; } -#else - ASSERT(0); -#endif - - numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -86,9 +77,8 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow * } #endif -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, @@ -100,6 +90,7 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *ro return 0; } +#if 0 int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; @@ -167,6 +158,46 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { } } + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} +#endif + +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { + ASSERT(pMsg != NULL); + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; + int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); + + terrno = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); + if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol); + if (nColData > 0) { + int32_t nRows = aColData[0].nVal; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t r = 0; r < nRows; ++r) { + if (tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now) < 0) { + return -1; + } + } + } + } else { + int32_t nRows = taosArrayGetSize(pData->aRowP); + for (int32_t r = 0; r < nRows; ++r) { + SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); + if (tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now) < 0) { + return -1; + } + } + } + } + if (terrno != TSDB_CODE_SUCCESS) return -1; return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 5adb2eb3592e7bcba7803b5e2972f0bdd3689adb..782cc69d30709c16a328cd056a8b52f6b42a5a6f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -115,7 +115,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; - if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; @@ -135,9 +134,6 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); } - // add tsdb page size config - if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; - return 0; } @@ -256,7 +252,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { } tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); - if (code < 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; + if (code < 0 || pCfg->tsdbPageSize < TSDB_MIN_PAGESIZE_PER_VNODE * 1024) { + pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; + } return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 4bdaf8d3531aefcbcd74eddecbc55152cfcbc798..4daab074b570a4acffcfaf976b2e7358f697eccd 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -14,14 +14,12 @@ */ #include "vnd.h" +#include "vnodeInt.h" -#define VND_INFO_FNAME "vnode.json" #define VND_INFO_FNAME_TMP "vnode_tmp.json" -static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); -static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo); -static int vnodeCommitImpl(void *arg); -static void vnodeWaitCommit(SVnode *pVnode); +static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); +static int vnodeCommitImpl(SCommitInfo *pInfo); int vnodeBegin(SVnode *pVnode) { // alloc buffer pool @@ -40,7 +38,7 @@ int vnodeBegin(SVnode *pVnode) { pVnode->state.commitID++; // begin meta - if (metaBegin(pVnode->pMeta, 0) < 0) { + if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } @@ -107,7 +105,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { // free info binary taosMemoryFree(data); - vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, pInfo->config.syncCfg.replicaNum); + vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname, + pInfo->config.syncCfg.replicaNum); return 0; @@ -185,29 +184,77 @@ _err: return -1; } +static void vnodePrepareCommit(SVnode *pVnode) { + tsem_wait(&pVnode->canCommit); + + tsdbPrepareCommit(pVnode->pTsdb); + metaPrepareAsyncCommit(pVnode->pMeta); + smaPrepareAsyncCommit(pVnode->pSma); + + vnodeBufPoolUnRef(pVnode->inUse); + pVnode->inUse = NULL; +} +static int32_t vnodeCommitTask(void *arg) { + int32_t code = 0; + + SCommitInfo *pInfo = (SCommitInfo *)arg; + + // commit + code = vnodeCommitImpl(pInfo); + if (code) goto _exit; + + // end commit + tsem_post(&pInfo->pVnode->canCommit); + +_exit: + taosMemoryFree(pInfo); + return code; +} int vnodeAsyncCommit(SVnode *pVnode) { - vnodeWaitCommit(pVnode); + int32_t code = 0; - // vnodeBufPoolSwitch(pVnode); - // tsdbPrepareCommit(pVnode->pTsdb); + // prepare to commit + vnodePrepareCommit(pVnode); - vnodeScheduleTask(vnodeCommitImpl, pVnode); + // schedule the task + pVnode->state.commitTerm = pVnode->state.applyTerm; - return 0; + SCommitInfo *pInfo = (SCommitInfo *)taosMemoryCalloc(1, sizeof(*pInfo)); + if (NULL == pInfo) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pInfo->info.config = pVnode->config; + pInfo->info.state.committed = pVnode->state.applied; + pInfo->info.state.commitTerm = pVnode->state.applyTerm; + pInfo->info.state.commitID = pVnode->state.commitID; + pInfo->pVnode = pVnode; + pInfo->txn = metaGetTxn(pVnode->pMeta); + vnodeScheduleTask(vnodeCommitTask, pInfo); + +_exit: + if (code) { + vError("vgId:%d %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), + pVnode->state.commitID); + } else { + vDebug("vgId:%d %s done", TD_VID(pVnode), __func__); + } + return code; } int vnodeSyncCommit(SVnode *pVnode) { vnodeAsyncCommit(pVnode); - vnodeWaitCommit(pVnode); - tsem_post(&(pVnode->canCommit)); + tsem_wait(&pVnode->canCommit); + tsem_post(&pVnode->canCommit); return 0; } -int vnodeCommit(SVnode *pVnode) { - int32_t code = 0; - int32_t lino = 0; - SVnodeInfo info = {0}; - char dir[TSDB_FILENAME_LEN]; +static int vnodeCommitImpl(SCommitInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; + + char dir[TSDB_FILENAME_LEN] = {0}; + SVnode *pVnode = pInfo->pVnode; vInfo("vgId:%d, start to commit, commit ID:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pVnode->state.commitID, pVnode->state.applied, pVnode->state.applyTerm); @@ -218,19 +265,13 @@ int vnodeCommit(SVnode *pVnode) { return -1; } - pVnode->state.commitTerm = pVnode->state.applyTerm; - // save info - info.config = pVnode->config; - info.state.committed = pVnode->state.applied; - info.state.commitTerm = pVnode->state.applyTerm; - info.state.commitID = pVnode->state.commitID; if (pVnode->pTfs) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); } else { snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); } - if (vnodeSaveInfo(dir, &info) < 0) { + if (vnodeSaveInfo(dir, &pInfo->info) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -238,23 +279,12 @@ int vnodeCommit(SVnode *pVnode) { // walBeginSnapshot(pVnode->pWal, pVnode->state.applied); syncBeginSnapshot(pVnode->sync, pVnode->state.applied); - code = smaPreCommit(pVnode->pSma); - TSDB_CHECK_CODE(code, lino, _exit); - - vnodeBufPoolUnRef(pVnode->inUse); - pVnode->inUse = NULL; - // commit each sub-system - if (metaCommit(pVnode->pMeta) < 0) { - code = TSDB_CODE_FAILED; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbCommit(pVnode->pTsdb); + code = tsdbCommit(pVnode->pTsdb, pInfo); TSDB_CHECK_CODE(code, lino, _exit); if (VND_IS_RSMA(pVnode)) { - code = smaCommit(pVnode->pSma); + code = smaCommit(pVnode->pSma, pInfo); TSDB_CHECK_CODE(code, lino, _exit); } @@ -264,7 +294,7 @@ int vnodeCommit(SVnode *pVnode) { } // commit info - if (vnodeCommitInfo(dir, &info) < 0) { + if (vnodeCommitInfo(dir, &pInfo->info) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } @@ -277,19 +307,18 @@ int vnodeCommit(SVnode *pVnode) { TSDB_CHECK_CODE(code, lino, _exit); } - if (metaFinishCommit(pVnode->pMeta) < 0) { + if (metaFinishCommit(pVnode->pMeta, pInfo->txn) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } - pVnode->state.committed = info.state.committed; + pVnode->state.committed = pInfo->info.state.committed; if (smaPostCommit(pVnode->pSma) < 0) { vError("vgId:%d, failed to post-commit sma since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } - // apply the commit (TODO) // walEndSnapshot(pVnode->pWal); syncEndSnapshot(pVnode->sync); @@ -318,20 +347,6 @@ void vnodeRollback(SVnode *pVnode) { (void)taosRemoveFile(tFName); } -static int vnodeCommitImpl(void *arg) { - SVnode *pVnode = (SVnode *)arg; - - // metaCommit(pVnode->pMeta); - tqCommit(pVnode->pTq); - // tsdbCommit(pVnode->pTsdb, ); - - // vnodeBufPoolRecycle(pVnode); - tsem_post(&(pVnode->canCommit)); - return 0; -} - -static FORCE_INLINE void vnodeWaitCommit(SVnode *pVnode) { tsem_wait(&pVnode->canCommit); } - static int vnodeEncodeState(const void *pObj, SJson *pJson) { const SVState *pState = (SVState *)pObj; @@ -390,7 +405,7 @@ _err: return -1; } -static int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) { +int vnodeDecodeInfo(uint8_t *pData, SVnodeInfo *pInfo) { SJson *pJson = NULL; pJson = tjsonParse(pData); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 0ff4a46d4418dd4156876c61b6e0286ccf01dd26..e09fafb75690f13dd160763126d139dd21ae8c1b 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -242,14 +242,14 @@ _err: return NULL; } -void vnodePreClose(SVnode *pVnode) { +void vnodePreClose(SVnode *pVnode) { vnodeQueryPreClose(pVnode); - vnodeSyncPreClose(pVnode); + vnodeSyncPreClose(pVnode); } void vnodeClose(SVnode *pVnode) { if (pVnode) { - vnodeCommit(pVnode); + vnodeSyncCommit(pVnode); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index e8cdf9513f8fc80d479114273a5bf4984bb71337..40705e553b3d22192c8d10b7d2bf292e2dce5fec 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -21,6 +21,8 @@ struct SVSnapReader { int64_t sver; int64_t ever; int64_t index; + // config + int8_t cfgDone; // meta int8_t metaDone; SMetaSnapReader *pMetaReader; @@ -88,6 +90,53 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t code = 0; + // CONFIG ============== + // FIXME: if commit multiple times and the config changed? + if (!pReader->cfgDone) { + char fName[TSDB_FILENAME_LEN]; + if (pReader->pVnode->pTfs) { + snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s%s%s", tfsGetPrimaryPath(pReader->pVnode->pTfs), TD_DIRSEP, + pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); + } else { + snprintf(fName, TSDB_FILENAME_LEN, "%s%s%s", pReader->pVnode->path, TD_DIRSEP, VND_INFO_FNAME); + } + + TdFilePtr pFile = taosOpenFile(fName, TD_FILE_READ); + if (NULL == pFile) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + int64_t size; + if (taosFStatFile(pFile, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFile); + goto _err; + } + + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size + 1); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFile); + goto _err; + } + ((SSnapDataHdr *)(*ppData))->type = SNAP_DATA_CFG; + ((SSnapDataHdr *)(*ppData))->size = size + 1; + ((SSnapDataHdr *)(*ppData))->data[size] = '\0'; + + if (taosReadFile(pFile, ((SSnapDataHdr *)(*ppData))->data, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFree(*ppData); + taosCloseFile(&pFile); + goto _err; + } + + taosCloseFile(&pFile); + + pReader->cfgDone = 1; + goto _exit; + } + // META ============== if (!pReader->metaDone) { // open reader if not @@ -230,6 +279,8 @@ struct SVSnapWriter { int64_t ever; int64_t commitID; int64_t index; + // config + SVnodeInfo info; // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb @@ -248,6 +299,10 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr int32_t code = 0; SVSnapWriter *pWriter = NULL; + // commit memory data + vnodeAsyncCommit(pVnode); + tsem_wait(&pVnode->canCommit); + // alloc pWriter = (SVSnapWriter *)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { @@ -258,16 +313,8 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr pWriter->sver = sver; pWriter->ever = ever; - // commit it - code = vnodeCommit(pVnode); - if (code) { - taosMemoryFree(pWriter); - goto _err; - } - // inc commit ID - pVnode->state.commitID++; - pWriter->commitID = pVnode->state.commitID; + pWriter->commitID = ++pVnode->state.commitID; vInfo("vgId:%d, vnode snapshot writer opened, sver:%" PRId64 " ever:%" PRId64 " commit id:%" PRId64, TD_VID(pVnode), sver, ever, pWriter->commitID); @@ -284,53 +331,89 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * int32_t code = 0; SVnode *pVnode = pWriter->pVnode; + // prepare + if (pWriter->pTsdbSnapWriter) { + tsdbSnapWriterPrepareClose(pWriter->pTsdbSnapWriter); + } + + // commit json + if (!rollback) { + pVnode->config = pWriter->info.config; + pVnode->state = (SVState){.committed = pWriter->info.state.committed, + .applied = pWriter->info.state.committed, + .commitID = pWriter->commitID, + .commitTerm = pWriter->info.state.commitTerm, + .applyTerm = pWriter->info.state.commitTerm}; + pVnode->statis = pWriter->info.statis; + char dir[TSDB_FILENAME_LEN] = {0}; + if (pWriter->pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); + } + + vnodeCommitInfo(dir, &pWriter->info); + } else { + vnodeRollback(pWriter->pVnode); + } + + // commit/rollback sub-system if (pWriter->pMetaSnapWriter) { code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback); - if (code) goto _err; + if (code) goto _exit; } if (pWriter->pTsdbSnapWriter) { code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); - if (code) goto _err; + if (code) goto _exit; } if (pWriter->pRsmaSnapWriter) { code = rsmaSnapWriterClose(&pWriter->pRsmaSnapWriter, rollback); - if (code) goto _err; + if (code) goto _exit; } - if (!rollback) { - SVnodeInfo info = {0}; - char dir[TSDB_FILENAME_LEN]; - - pVnode->state.committed = pWriter->ever; - pVnode->state.applied = pWriter->ever; - pVnode->state.applyTerm = pSnapshot->lastApplyTerm; - pVnode->state.commitTerm = pSnapshot->lastApplyTerm; - - info.config = pVnode->config; - info.state.committed = pVnode->state.applied; - info.state.commitTerm = pVnode->state.applyTerm; - info.state.commitID = pVnode->state.commitID; - snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); - code = vnodeSaveInfo(dir, &info); - if (code) goto _err; - - code = vnodeCommitInfo(dir, &info); - if (code) goto _err; - - vnodeBegin(pVnode); - } else { - ASSERT(0); - } + vnodeBegin(pVnode); _exit: - vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); - taosMemoryFree(pWriter); + if (code) { + vError("vgId:%d, vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); + } else { + vInfo("vgId:%d, vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); + taosMemoryFree(pWriter); + } + tsem_post(&pVnode->canCommit); return code; +} -_err: - vError("vgId:%d, vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); +static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { + int32_t code = 0; + + SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; + + // decode info + if (vnodeDecodeInfo(pHdr->data, &pWriter->info) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + // change some value + pWriter->info.state.commitID = pWriter->commitID; + + // modify info as needed + char dir[TSDB_FILENAME_LEN] = {0}; + if (pWriter->pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pWriter->pVnode->pTfs), TD_DIRSEP, + pWriter->pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); + } + if (vnodeSaveInfo(dir, &pWriter->info) < 0) { + code = terrno; + goto _exit; + } + +_exit: return code; } @@ -347,6 +430,10 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { pHdr->type, nData); switch (pHdr->type) { + case SNAP_DATA_CFG: { + code = vnodeSnapWriteInfo(pWriter, pData, nData); + if (code) goto _err; + } break; case SNAP_DATA_META: { // meta if (pWriter->pMetaSnapWriter == NULL) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 13a595d8181dad1b52dc534bb22cfff58b6bfc6e..9dc1783fc79e24610493434d0a61a4cc0bedc582 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -101,6 +101,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { } if (flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + // SVCreateTbReq if (tStartDecode(&dc) < 0) { code = TSDB_CODE_INVALID_MSG; goto _err; @@ -126,12 +127,22 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { *(int64_t *)(dc.data + dc.pos + 8) = ctime; tEndDecode(&dc); + + // SSubmitTbData + int64_t suid; + if (tDecodeI64(&dc, &suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _err; + } + + *(int64_t *)(dc.data + dc.pos) = uid; } tEndDecode(&dc); } tEndDecode(&dc); + tDecoderClear(&dc); } break; case TDMT_VND_DELETE: { int32_t size; @@ -190,17 +201,22 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); + terrno = TSDB_CODE_VND_DUP_REQUEST; pRsp->info.handle = NULL; return -1; } vDebug("vgId:%d, start to process write request %s, index:%" PRId64, TD_VID(pVnode), TMSG_INFO(pMsg->msgType), version); + ASSERT(pVnode->state.applyTerm <= pMsg->info.conn.applyTerm); + ASSERT(pVnode->state.applied + 1 == version); pVnode->state.applied = version; pVnode->state.applyTerm = pMsg->info.conn.applyTerm; + if (!syncUtilUserCommit(pMsg->msgType)) goto _exit; + // skip header pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); len = pMsg->contLen - sizeof(SMsgHead); @@ -236,7 +252,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp break; /* TSDB */ case TDMT_VND_SUBMIT: - if (vnodeProcessSubmitReq(pVnode, version, pMsg->pCont, pMsg->contLen, pRsp) < 0) goto _err; + if (vnodeProcessSubmitReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; case TDMT_VND_DELETE: if (vnodeProcessDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; @@ -300,7 +316,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_COMMIT: - goto _do_commit; + vnodeSyncCommit(pVnode); + vnodeBegin(pVnode); + goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return -1; @@ -318,13 +336,8 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (vnodeShouldCommit(pVnode)) { - _do_commit: vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); - // commit current change - if (vnodeCommit(pVnode) < 0) { - vError("vgId:%d, failed to commit vnode since %s.", TD_VID(pVnode), tstrerror(terrno)); - goto _err; - } + vnodeAsyncCommit(pVnode); // start a new one if (vnodeBegin(pVnode) < 0) { @@ -333,6 +346,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } } +_exit: return 0; _err: @@ -353,7 +367,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); // if ((pMsg->msgType == TDMT_SCH_QUERY) && !vnodeIsLeader(pVnode)) { if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } @@ -376,12 +390,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { pMsg->msgType == TDMT_VND_BATCH_META) && !syncIsReadyForRead(pVnode->sync)) { // !vnodeIsLeader(pVnode)) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { - vnodeRedirectRpcMsg(pVnode, pMsg); + vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); return 0; } @@ -857,6 +871,7 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { #if 1 int32_t code = 0; + terrno = 0; SSubmitReq2 *pSubmitReq = &(SSubmitReq2){0}; SSubmitRsp2 *pSubmitRsp = &(SSubmitRsp2){0}; @@ -868,7 +883,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq // decode SDecoder dc = {0}; - tDecoderInit(&dc, (char *)pReq + sizeof(SMsgHead), len - sizeof(SMsgHead)); + tDecoderInit(&dc, pReq, len); if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; @@ -876,6 +891,11 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq tDecoderClear(&dc); // check + code = tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq); + if (code) { + goto _exit; + } + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); @@ -887,6 +907,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq code = metaGetInfo(pVnode->pMeta, pSubmitTbData->uid, &info, NULL); if (code) { code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), pSubmitTbData->uid); goto _exit; } @@ -946,7 +967,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (pSubmitRsp->aCreateTbRsp == NULL && (pSubmitRsp->aCreateTbRsp = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(SVCreateTbRsp))) == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -958,7 +979,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (newTbUids == NULL && (newTbUids = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(int64_t))) == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } @@ -1005,6 +1026,7 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } // clear @@ -1012,17 +1034,19 @@ _exit: tDestroySSubmitReq2(pSubmitReq, TSDB_MSG_FLG_DECODE); tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); + if (code) terrno = code; + return code; #else SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; + SSubmitRsp submitRsp = {0}; + int32_t nRows = 0; + int32_t tsize, ret; + SEncoder encoder = {0}; + SArray *newTbUids = NULL; + SVStatis statis = {0}; + bool tbCreated = false; terrno = TSDB_CODE_SUCCESS; pRsp->code = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 7f5b9687fa97483c4063d0d0f723032e2d86509d..2c23646db1c874a0cbb671784fbcdcec0e5918ec 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -16,28 +16,15 @@ #define _DEFAULT_SOURCE #include "vnd.h" -#define BATCH_DISABLE 1 - -static inline bool vnodeIsMsgBlock(tmsg_t type) { - return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL); -} +#define BATCH_ENABLE 0 static inline bool vnodeIsMsgWeak(tmsg_t type) { return false; } static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { - if (vnodeIsMsgBlock(pMsg->msgType)) { - const STraceId *trace = &pMsg->info.traceId; - taosThreadMutexLock(&pVnode->lock); - if (!pVnode->blocked) { - vGTrace("vgId:%d, msg:%p wait block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); - pVnode->blocked = true; - taosThreadMutexUnlock(&pVnode->lock); - tsem_wait(&pVnode->syncSem); - } else { - taosThreadMutexUnlock(&pVnode->lock); - } - } + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p wait block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq); + tsem_wait(&pVnode->syncSem); } static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { @@ -45,15 +32,18 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) { const STraceId *trace = &pMsg->info.traceId; taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { - vGTrace("vgId:%d, msg:%p post block, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); + vGTrace("vgId:%d, msg:%p post block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq); pVnode->blocked = false; + pVnode->blockSec = 0; + pVnode->blockSeq = 0; tsem_post(&pVnode->syncSem); } taosThreadMutexUnlock(&pVnode->lock); } } -void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { +void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { SEpSet newEpSet = {0}; syncGetRetryEpSet(pVnode->sync, &newEpSet); @@ -66,8 +56,20 @@ void vnodeRedirectRpcMsg(SVnode *pVnode, SRpcMsg *pMsg) { } pMsg->info.hasEpSet = 1; - SRpcMsg rsp = {.code = TSDB_CODE_SYN_NOT_LEADER, .info = pMsg->info, .msgType = pMsg->msgType + 1}; - tmsgSendRedirectRsp(&rsp, &newEpSet); + if (code == 0) code = TSDB_CODE_SYN_NOT_LEADER; + + SRpcMsg rsp = {.code = code, .info = pMsg->info, .msgType = pMsg->msgType + 1}; + int32_t contLen = tSerializeSEpSet(NULL, 0, &newEpSet); + + rsp.pCont = rpcMallocCont(contLen); + if (rsp.pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSEpSet(rsp.pCont, contLen, &newEpSet); + rsp.contLen = contLen; + } + + tmsgSendRsp(&rsp); } static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -87,8 +89,8 @@ static void inline vnodeHandleWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) { - if (code == TSDB_CODE_SYN_NOT_LEADER) { - vnodeRedirectRpcMsg(pVnode, pMsg); + if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_RESTORING) { + vnodeRedirectRpcMsg(pVnode, pMsg, code); } else { const STraceId *trace = &pMsg->info.traceId; vGError("vgId:%d, msg:%p failed to propose since %s, code:0x%x", pVnode->config.vgId, pMsg, tstrerror(code), code); @@ -99,28 +101,35 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code) } } +#if BATCH_ENABLE + static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool *pIsWeakArr, int32_t *arrSize) { if (*arrSize <= 0) return; + SRpcMsg *pLastMsg = pMsgArr[*arrSize - 1]; -#if BATCH_DISABLE - int32_t code = syncPropose(pVnode->sync, pMsgArr[0], pIsWeakArr[0]); -#else + taosThreadMutexLock(&pVnode->lock); int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize); -#endif + bool wait = (code == 0 && vnodeIsBlockMsg(pLastMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + } + taosThreadMutexUnlock(&pVnode->lock); if (code > 0) { for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleWriteMsg(pVnode, pMsgArr[i]); } - } else if (code == 0) { - vnodeWaitBlockMsg(pVnode, pMsgArr[*arrSize - 1]); - } else { + } else if (code < 0) { if (terrno != 0) code = terrno; for (int32_t i = 0; i < *arrSize; ++i) { vnodeHandleProposeError(pVnode, pMsgArr[i], code); } } + if (wait) vnodeWaitBlockMsg(pVnode, pLastMsg); + pLastMsg = NULL; + for (int32_t i = 0; i < *arrSize; ++i) { SRpcMsg *pMsg = pMsgArr[i]; const STraceId *trace = &pMsg->info.traceId; @@ -153,8 +162,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) if (!pVnode->restored) { vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); - terrno = TSDB_CODE_APP_NOT_READY; - vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_APP_NOT_READY); + terrno = TSDB_CODE_SYN_RESTORING; + vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); continue; @@ -177,7 +186,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - if (isBlock || BATCH_DISABLE) { + if (isBlock) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } @@ -185,7 +194,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) pIsWeakArr[arrayPos] = isWeak; arrayPos++; - if (isBlock || msg == numOfMsgs - 1 || BATCH_DISABLE) { + if (isBlock || msg == numOfMsgs - 1) { vnodeProposeBatchMsg(pVnode, pMsgArr, pIsWeakArr, &arrayPos); } } @@ -194,6 +203,79 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) taosMemoryFree(pIsWeakArr); } +#else + +static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) { + int64_t seq = 0; + + taosThreadMutexLock(&pVnode->lock); + int32_t code = syncPropose(pVnode->sync, pMsg, isWeak, &seq); + bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType)); + if (wait) { + ASSERT(!pVnode->blocked); + pVnode->blocked = true; + pVnode->blockSec = taosGetTimestampSec(); + pVnode->blockSeq = seq; +#if 0 + pVnode->blockInfo = pMsg->info; +#endif + } + taosThreadMutexUnlock(&pVnode->lock); + + if (code > 0) { + vnodeHandleWriteMsg(pVnode, pMsg); + } else if (code < 0) { + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + } + + if (wait) vnodeWaitBlockMsg(pVnode, pMsg); + return code; +} + +void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { + SVnode *pVnode = pInfo->ahandle; + int32_t vgId = pVnode->config.vgId; + int32_t code = 0; + SRpcMsg *pMsg = NULL; + vTrace("vgId:%d, get %d msgs from vnode-write queue", vgId, numOfMsgs); + + for (int32_t msg = 0; msg < numOfMsgs; msg++) { + if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; + bool isWeak = vnodeIsMsgWeak(pMsg->msgType); + + const STraceId *trace = &pMsg->info.traceId; + vGTrace("vgId:%d, msg:%p get from vnode-write queue, weak:%d block:%d msg:%d:%d, handle:%p", vgId, pMsg, isWeak, + vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); + + if (!pVnode->restored) { + vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); + vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodePreProcessWriteMsg(pVnode, pMsg); + if (code != 0) { + vGError("vgId:%d, msg:%p failed to pre-process since %s", vgId, pMsg, terrstr()); + if (terrno != 0) code = terrno; + vnodeHandleProposeError(pVnode, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + continue; + } + + code = vnodeProposeMsg(pVnode, pMsg, isWeak); + + vGTrace("vgId:%d, msg:%p is freed, code:0x%x", vgId, pMsg, code); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); + } +} + +#endif + void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnode *pVnode = pInfo->ahandle; int32_t vgId = pVnode->config.vgId; @@ -203,8 +285,16 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { for (int32_t i = 0; i < numOfMsgs; ++i) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; const STraceId *trace = &pMsg->info.traceId; - vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg, - TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex); + + if (vnodeIsMsgBlock(pMsg->msgType)) { + vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64 + ", blocking msg obtained sec:%d seq:%" PRId64, + vgId, pMsg, TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex, pVnode->blockSec, + pVnode->blockSeq); + } else { + vGTrace("vgId:%d, msg:%p get from vnode-apply queue, type:%s handle:%p index:%" PRId64, vgId, pMsg, + TMSG_INFO(pMsg->msgType), pMsg->info.handle, pMsg->info.conn.applyIndex); + } SRpcMsg rsp = {.code = pMsg->code, .info = pMsg->info}; if (rsp.code == 0) { @@ -295,7 +385,7 @@ static int32_t vnodeSyncGetSnapshot(const SSyncFSM *pFsm, SSnapshot *pSnapshot) return 0; } -static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { SVnode *pVnode = pFsm->data; pMsg->info.conn.applyIndex = pMeta->index; pMsg->info.conn.applyTerm = pMeta->term; @@ -306,17 +396,18 @@ static void vnodeSyncApplyMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbM pVnode->config.vgId, pFsm, pMeta->index, pMeta->term, pMsg->info.conn.applyIndex, pMeta->isWeak, pMeta->code, pMeta->state, syncStr(pMeta->state), TMSG_INFO(pMsg->msgType)); - tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); + return tmsgPutToQueue(&pVnode->msgCb, APPLY_QUEUE, pMsg); } -static void vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); +static int32_t vnodeSyncCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } -static void vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { +static int32_t vnodeSyncPreCommitMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { if (pMeta->isWeak == 1) { - vnodeSyncApplyMsg(pFsm, pMsg, pMeta); + return vnodeSyncApplyMsg(pFsm, pMsg, pMeta); } + return 0; } static void vnodeSyncRollBackMsg(const SSyncFSM *pFsm, SRpcMsg *pMsg, const SFsmCbMeta *pMeta) { @@ -514,16 +605,7 @@ void vnodeSyncPreClose(SVnode *pVnode) { vInfo("vgId:%d, pre close sync", pVnode->config.vgId); syncLeaderTransfer(pVnode->sync); syncPreStop(pVnode->sync); -#if 0 - while (syncSnapshotRecving(pVnode->sync)) { - vInfo("vgId:%d, snapshot is recving", pVnode->config.vgId); - taosMsleep(300); - } - while (syncSnapshotSending(pVnode->sync)) { - vInfo("vgId:%d, snapshot is sending", pVnode->config.vgId); - taosMsleep(300); - } -#endif + taosThreadMutexLock(&pVnode->lock); if (pVnode->blocked) { vInfo("vgId:%d, post block after close sync", pVnode->config.vgId); @@ -538,27 +620,55 @@ void vnodeSyncClose(SVnode *pVnode) { syncStop(pVnode->sync); } +void vnodeSyncCheckTimeout(SVnode *pVnode) { + vTrace("vgId:%d, check sync timeout msg", pVnode->config.vgId); + taosThreadMutexLock(&pVnode->lock); + if (pVnode->blocked) { + int32_t curSec = taosGetTimestampSec(); + int32_t delta = curSec - pVnode->blockSec; + if (delta > VNODE_TIMEOUT_SEC) { + vError("vgId:%d, failed to propose since timeout and post block, start:%d cur:%d delta:%d seq:%" PRId64, + pVnode->config.vgId, pVnode->blockSec, curSec, delta, pVnode->blockSeq); + if (syncSendTimeoutRsp(pVnode->sync, pVnode->blockSeq) != 0) { +#if 0 + SRpcMsg rpcMsg = {.code = TSDB_CODE_SYN_TIMEOUT, .info = pVnode->blockInfo}; + vError("send timeout response since its applyed, seq:%" PRId64 " handle:%p ahandle:%p", pVnode->blockSeq, + rpcMsg.info.handle, rpcMsg.info.ahandle); + rpcSendResponse(&rpcMsg); +#endif + } + pVnode->blocked = false; + pVnode->blockSec = 0; + pVnode->blockSeq = 0; + tsem_post(&pVnode->syncSem); + } + } + taosThreadMutexUnlock(&pVnode->lock); +} + bool vnodeIsRoleLeader(SVnode *pVnode) { SSyncState state = syncGetState(pVnode->sync); return state.state == TAOS_SYNC_STATE_LEADER; } bool vnodeIsLeader(SVnode *pVnode) { + terrno = 0; SSyncState state = syncGetState(pVnode->sync); - if (state.state != TAOS_SYNC_STATE_LEADER || !state.restored) { - if (state.state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } - vInfo("vgId:%d, vnode not ready, state:%s, restore:%d", pVnode->config.vgId, syncStr(state.state), state.restored); + if (terrno != 0) { + vInfo("vgId:%d, vnode is stopping", pVnode->config.vgId); + return false; + } + + if (state.state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + vInfo("vgId:%d, vnode not leader, state:%s", pVnode->config.vgId, syncStr(state.state)); return false; } - if (!pVnode->restored) { - vInfo("vgId:%d, vnode not restored", pVnode->config.vgId); - terrno = TSDB_CODE_APP_NOT_READY; + if (!state.restored || !pVnode->restored) { + terrno = TSDB_CODE_SYN_RESTORING; + vInfo("vgId:%d, vnode not restored:%d:%d", pVnode->config.vgId, state.restored, pVnode->restored); return false; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c95d0fe4620f7a068813701588da77d9d149a936..6e072a96309f79d4a824ded468a0e5b258e4e3e8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -547,6 +547,14 @@ typedef struct SCtgOperation { #define ctgDebug(param, ...) qDebug("CTG:%p " param, pCtg, __VA_ARGS__) #define ctgTrace(param, ...) qTrace("CTG:%p " param, pCtg, __VA_ARGS__) +#define ctgTaskFatal(param, ...) qFatal("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskError(param, ...) qError("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskWarn(param, ...) qWarn("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskInfo(param, ...) qInfo("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskDebug(param, ...) qDebug("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) +#define ctgTaskTrace(param, ...) qTrace("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) + + #define CTG_LOCK_DEBUG(...) \ do { \ if (gCTGDebug.lockEnable) { \ @@ -762,7 +770,6 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) int32_t ctgGenerateVgList(SCatalog* pCtg, SHashObj* vgHash, SArray** pList); void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); -void ctgFreeVgInfo(SDBVgInfo* vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); @@ -789,6 +796,11 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); int32_t ctgdGetOneHandle(SCatalog **pHandle); +int ctgVgInfoComp(const void* lp, const void* rp); +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); +int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb); +int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName); +void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3a398d15513eeda063448373b44bfeca10025d15..e9e0bae8d7c569760d4dfd74aa0a1d1a15753f7c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -505,8 +505,7 @@ _return: taosMemoryFreeClear(tbMeta); if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } if (vgList) { @@ -546,13 +545,41 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_RET(code); } +int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { + int32_t code = 0; + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + + CTG_ERR_RET(ctgAcquireVgMetaFromCache(pCtg, db, pTableName->tname, &dbCache, &tbCache)); + + if (NULL == dbCache || NULL == tbCache) { + *pTableMeta = NULL; + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pTableName, pVgroup)); + + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)pTableName; + ctx.flag = CTG_FLAG_UNKNOWN_STB; + CTG_ERR_JRET(ctgCopyTbMeta(pCtg, &ctx, &dbCache, &tbCache, pTableMeta, db)); + +_return: + + ctgReleaseVgMetaToCache(pCtg, dbCache, tbCache); + + CTG_RET(code); +} + + int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName) { int32_t code = 0; @@ -619,7 +646,7 @@ int32_t catalogInit(SCatalogCfg* cfg) { gCtgMgmt.queue.head = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == gCtgMgmt.queue.head) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); - CTG_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } gCtgMgmt.queue.tail = gCtgMgmt.queue.head; @@ -778,8 +805,7 @@ _return: } if (vgInfo) { - taosHashCleanup(vgInfo->vgHash); - taosMemoryFreeClear(vgInfo); + freeVgInfo(vgInfo); } CTG_API_LEAVE(code); @@ -836,8 +862,7 @@ _return: ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } else if (dbInfo) { - taosHashCleanup(dbInfo->vgHash); - taosMemoryFreeClear(dbInfo); + freeVgInfo(dbInfo); } CTG_API_LEAVE(code); @@ -849,7 +874,7 @@ int32_t catalogUpdateDBVgInfo(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t code = 0; if (NULL == pCtg || NULL == dbFName || NULL == dbInfo) { - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1122,6 +1147,13 @@ int32_t catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableName, CTG_API_LEAVE(ctgGetTbHashVgroup(pCtg, NULL, pTableName, pVgroup, exists)); } +int32_t catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { + CTG_API_ENTER(); + + CTG_API_LEAVE(ctgGetCachedTbVgMeta(pCtg, pTableName, pVgroup, pTableMeta)); +} + + #if 0 int32_t catalogGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 7138e0ce16a4c3543bf145251640447fd986e54d..5b01ac1fb96007234863d7c96f7aaa1e75b3c962 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1094,6 +1094,9 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } + if (code) { + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, tstrerror(code)); + } if (pTask->res || code) { ctgHandleTaskEnd(pTask, code); } @@ -1124,7 +1127,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SVgroupInfo vgInfo = {0}; CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + ctgTaskDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); @@ -1144,7 +1147,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SVgroupInfo vgInfo = {0}; CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + ctgTaskDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); @@ -1162,7 +1165,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu return TSDB_CODE_SUCCESS; } - ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); @@ -1180,7 +1183,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgTaskError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1190,7 +1193,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu } if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); + ctgTaskDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); taosMemoryFreeClear(pOut->tbMeta); @@ -1207,11 +1210,11 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMeta* stbMeta = NULL; (void)ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta); if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) { - ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); + ctgTaskDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; taosMemoryFreeClear(stbMeta); } else { - ctgDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName)); + ctgTaskDebug("need to get/update stb meta, tbName:%s", tNameGetTableName(pName)); taosMemoryFreeClear(pOut->tbMeta); taosMemoryFreeClear(stbMeta); } @@ -1225,7 +1228,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu break; } default: - ctgError("invalid reqType %d", reqType); + ctgTaskError("invalid reqType %d", reqType); CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); } @@ -1280,6 +1283,7 @@ _return: TSWAP(pTask->res, ctx->pResList); taskDone = true; } + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, tstrerror(code)); } if (pTask->res && taskDone) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 19b7ee32aebc05c211d564e47fae3af7eaeb4f81..94ee295fe420ce9891c66b2431f836a036a107f3 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -130,7 +130,7 @@ void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { } void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) { - if (pCache) { + if (pCache && dbCache) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); } @@ -151,6 +151,18 @@ void ctgReleaseTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache } } +void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) { + if (pCache && dbCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); + } +} + int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -226,6 +238,76 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + bool vgInCache = false; + + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + CTG_CACHE_STAT_INC(numOfVgMiss, 1); + goto _return; + } + + ctgRLockVgInfo(pCtg, dbCache, &vgInCache); + if (!vgInCache) { + ctgDebug("vgInfo of db %s not in cache", dbFName); + CTG_CACHE_STAT_INC(numOfVgMiss, 1); + goto _return; + } + + *pDb = dbCache; + + CTG_CACHE_STAT_INC(numOfVgHit, 1); + + ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); + + tbCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == tbCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + CTG_CACHE_STAT_INC(numOfMetaMiss, 1); + goto _return; + } + + CTG_LOCK(CTG_READ, &tbCache->metaLock); + if (NULL == tbCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", tbName, dbFName); + CTG_CACHE_STAT_INC(numOfMetaMiss, 1); + goto _return; + } + + *pTb = tbCache; + + ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); + + CTG_CACHE_STAT_INC(numOfMetaHit, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + + if (vgInCache) { + ctgRUnlockVgInfo(dbCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } + + *pDb = NULL; + *pTb = NULL; + + return TSDB_CODE_SUCCESS; +} + + +/* int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; @@ -276,6 +358,49 @@ _return: return TSDB_CODE_SUCCESS; } +*/ + +int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { + SCtgTbCache *pCache = NULL; + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pTb = pCache; + + ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(numOfMetaHit, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(numOfMetaMiss, 1); + + *pTb = NULL; + + return TSDB_CODE_SUCCESS; +} int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; @@ -334,25 +459,9 @@ int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32 return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - SCtgTbCache *tbCache = NULL; - *pTableMeta = NULL; - - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); - } else { - tNameGetFullDbName(ctx->pName, dbFName); - } - - ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); - if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName) { + SCtgDBCache *dbCache = *pDb; + SCtgTbCache *tbCache = *pTb; STableMeta *tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; @@ -363,13 +472,11 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** int32_t metaSize = CTG_META_SIZE(tbMeta); *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == *pTableMeta) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*pTableMeta, tbMeta, metaSize); - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } @@ -379,39 +486,72 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** int32_t metaSize = sizeof(SCTableMeta); *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == *pTableMeta) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*pTableMeta, tbMeta, metaSize); - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + *pTb = NULL; + } + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); + *pDb = NULL; ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; } + *pTb = tbCache; + STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != ctx->tbInfo.suid) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%" PRIx64, stbMeta->suid, ctx->tbInfo.suid); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + taosMemoryFreeClear(*pTableMeta); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + *pTableMeta = NULL; + + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { + strcpy(dbFName, ctx->pName->dbname); + } else { + tNameGetFullDbName(ctx->pName, dbFName); + } + + ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgCopyTbMeta(pCtg, ctx, &dbCache, &tbCache, pTableMeta, dbFName)); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); @@ -460,12 +600,17 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, // PROCESS FOR CHILD TABLE - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + if (tbCache) { + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + } + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { - ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } @@ -794,7 +939,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); taosMemoryFree(op); - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -803,6 +948,14 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId dbFName = p + 1; } + code = ctgMakeVgArray(dbInfo); + if (code) { + taosMemoryFree(op); + taosMemoryFree(msg); + freeVgInfo(dbInfo); + CTG_ERR_RET(code); + } + tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName)); msg->pCtg = pCtg; msg->dbId = dbId; @@ -816,7 +969,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId _return: - ctgFreeVgInfo(dbInfo); + freeVgInfo(dbInfo); CTG_RET(code); } @@ -1549,6 +1702,20 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } +int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { + int32_t* key = (int32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->vgId) { + return -1; + } else if (*key > pVg->vgId) { + return 1; + } + + return 0; +} + + int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1561,7 +1728,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { } if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, + ctgDebug("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } @@ -1600,7 +1767,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { goto _return; } - ctgFreeVgInfo(vgInfo); + freeVgInfo(vgInfo); } vgCache->vgInfo = dbInfo; @@ -1621,7 +1788,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { _return: - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1674,7 +1841,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgCache.vgInfo); + freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); @@ -1930,7 +2097,13 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SVgroupInfo *pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); + ctgDebug("no vgroup %d in db %s vgHash, ignore epset update", msg->vgId, msg->dbFName); + goto _return; + } + + SVgroupInfo *pInfo2 = taosArraySearch(vgInfo->vgArray, &msg->vgId, ctgVgInfoIdComp, TD_EQ); + if (NULL == pInfo2) { + ctgDebug("no vgroup %d in db %s vgArray, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1941,6 +2114,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { msg->epSet.numOfEps, pNewEp->fqdn, pNewEp->port, msg->dbFName); pInfo->epSet = msg->epSet; + pInfo2->epSet = msg->epSet; _return: @@ -2057,7 +2231,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) { switch (op->opId) { case CTG_OP_UPDATE_VGROUP: { SCtgUpdateVgMsg *msg = op->data; - ctgFreeVgInfo(msg->dbInfo); + freeVgInfo(msg->dbInfo); taosMemoryFreeClear(op->data); break; } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 753c4ce9171a7acc6a1b5d065751b0ca32268972..7cc6c90d303c4c425358800eb646f986417cefdd 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -384,13 +384,13 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); if (NULL == param) { qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); - CTG_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param->reqType = msgType; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f795be3ac2ab41ebb40e818584c3be59841995be..802ecde63eaae46645712a5d3f60ae76cdfeee6f 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -231,20 +231,7 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) { CTG_CACHE_STAT_DEC(numOfTbl, tblNum); } -void ctgFreeVgInfo(SDBVgInfo* vgInfo) { - if (NULL == vgInfo) { - return; - } - - if (vgInfo->vgHash) { - taosHashCleanup(vgInfo->vgHash); - vgInfo->vgHash = NULL; - } - - taosMemoryFreeClear(vgInfo); -} - -void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { ctgFreeVgInfo(dbCache->vgCache.vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); } void ctgFreeDbCache(SCtgDBCache* dbCache) { if (NULL == dbCache) { @@ -366,8 +353,7 @@ void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { } if (pOutput->dbVgroup) { - taosHashCleanup(pOutput->dbVgroup->vgHash); - taosMemoryFreeClear(pOutput->dbVgroup); + freeVgInfo(pOutput->dbVgroup); } taosMemoryFree(pOutput); @@ -573,8 +559,7 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void** pRes) { case CTG_TASK_GET_DB_VGROUP: { if (*pRes) { SDBVgInfo* pInfo = (SDBVgInfo*)*pRes; - taosHashCleanup(pInfo->vgHash); - taosMemoryFreeClear(*pRes); + freeVgInfo(pInfo); } break; } @@ -837,10 +822,36 @@ _return: CTG_RET(code); } +int ctgVgInfoComp(const void* lp, const void* rp) { + SVgroupInfo* pLeft = (SVgroupInfo*)lp; + SVgroupInfo* pRight = (SVgroupInfo*)rp; + if (pLeft->hashBegin < pRight->hashBegin) { + return -1; + } else if (pLeft->hashBegin > pRight->hashBegin) { + return 1; + } + + return 0; +} + +int32_t ctgHashValueComp(void const* lp, void const* rp) { + uint32_t* key = (uint32_t*)lp; + SVgroupInfo* pVg = (SVgroupInfo*)rp; + + if (*key < pVg->hashBegin) { + return -1; + } else if (*key > pVg->hashEnd) { + return 1; + } + + return 0; +} + int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup) { int32_t code = 0; + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -856,6 +867,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + +/* void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -867,10 +881,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } +*/ if (NULL == vgInfo) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, - taosHashGetSize(dbInfo->vgHash)); + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -883,37 +898,15 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName CTG_RET(code); } -int32_t ctgHashValueComp(void const* lp, void const* rp) { - uint32_t* key = (uint32_t*)lp; - SVgroupInfo* pVg = *(SVgroupInfo**)rp; - - if (*key < pVg->hashBegin) { - return -1; - } else if (*key > pVg->hashEnd) { - return 1; - } - - return 0; -} - -int ctgVgInfoComp(const void* lp, const void* rp) { - SVgroupInfo* pLeft = *(SVgroupInfo**)lp; - SVgroupInfo* pRight = *(SVgroupInfo**)rp; - if (pLeft->hashBegin < pRight->hashBegin) { - return -1; - } else if (pLeft->hashBegin > pRight->hashBegin) { - return 1; - } - - return 0; -} - int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; - int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + + CTG_ERR_RET(ctgMakeVgArray(dbInfo)); + + int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -923,20 +916,13 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - if (NULL == pIter) { - ctgError("empty vgHash, db:%s, vgroup number:%d", dbFName, vgNum); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { - taosHashCancelIterate(dbInfo->vgHash, pIter); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - *vgInfo = *(SVgroupInfo*)pIter; + *vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); @@ -951,19 +937,9 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosHashCancelIterate(dbInfo->vgHash, pIter); return TSDB_CODE_SUCCESS; } - SArray* pVgList = taosArrayInit(vgNum, POINTER_BYTES); - void* pIter = taosHashIterate(dbInfo->vgHash, NULL); - while (pIter) { - taosArrayPush(pVgList, &pIter); - pIter = taosHashIterate(dbInfo->vgHash, pIter); - } - - taosArraySort(pVgList, ctgVgInfoComp); - char tbFullName[TSDB_TABLE_FNAME_LEN]; sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); @@ -979,20 +955,15 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); - SVgroupInfo** p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); - - if (NULL == p) { + vgInfo = taosArraySearch(dbInfo->vgArray, &hashValue, ctgHashValueComp, TD_EQ); + if (NULL == vgInfo) { ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, - taosHashGetSize(dbInfo->vgHash)); - taosArrayDestroy(pVgList); + (int32_t)taosArrayGetSize(dbInfo->vgArray)); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - vgInfo = *p; - SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pNewVg) { - taosArrayDestroy(pVgList); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1012,8 +983,6 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* } } - taosArrayDestroy(pVgList); - CTG_RET(code); } @@ -1057,7 +1026,33 @@ int32_t ctgDbVgVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { + if (NULL == dbInfo) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgHash && NULL == dbInfo->vgArray) { + dbInfo->vgArray = taosArrayInit(100, sizeof(SVgroupInfo)); + if (NULL == dbInfo->vgArray) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + taosArrayPush(dbInfo->vgArray, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); + } + + taosArraySort(dbInfo->vgArray, ctgVgInfoComp); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { + CTG_ERR_RET(ctgMakeVgArray(src)); + *dst = taosMemoryMalloc(sizeof(SDBVgInfo)); if (NULL == *dst) { qError("malloc %d failed", (int32_t)sizeof(SDBVgInfo)); @@ -1090,6 +1085,10 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { pIter = taosHashIterate(src->vgHash, pIter); } + if (src->vgArray) { + (*dst)->vgArray = taosArrayDup(src->vgArray, NULL); + } + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 4d0c5389e145bc91ebe459f95d8a34fa5b1552df..6acf19218d893a9962853e1fe7b0c73c8dddf7bc 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -34,6 +34,7 @@ extern "C" { #define EXPLAIN_SYSTBL_SCAN_FORMAT "System Table Scan on %s" #define EXPLAIN_DISTBLK_SCAN_FORMAT "Block Dist Scan on %s" #define EXPLAIN_LASTROW_SCAN_FORMAT "Last Row Scan on %s" +#define EXPLAIN_TABLE_COUNT_SCAN_FORMAT "Table Count Row Scan on %s" #define EXPLAIN_PROJECTION_FORMAT "Projection" #define EXPLAIN_JOIN_FORMAT "%s" #define EXPLAIN_AGG_FORMAT "Aggragate" diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d58c4dc6d30142ce698a0c6659d1b8be7692dad1..5f1b87a1386c1f9fa2009242c1660adf52bdc6a8 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -36,7 +36,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; - (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); @@ -497,7 +497,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char* buf2 = taosMemoryMalloc(SHOW_CREATE_TB_RESULT_FIELD2_LEN); if (NULL == buf2) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 410e62a18b9d2b68e7976a3e9728a5e874dd09ee..253718048dcebb71b64ca4d4b48b3004c6d9ea3a 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -19,6 +19,7 @@ #include "query.h" #include "tcommon.h" #include "tdatablock.h" +#include "systable.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel); @@ -76,19 +77,19 @@ int32_t qExplainInitCtx(SExplainCtx **pCtx, SHashObj *groupHash, bool verbose, d SExplainCtx *ctx = taosMemoryCalloc(1, sizeof(SExplainCtx)); if (NULL == ctx) { qError("calloc SExplainCtx failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SArray *rows = taosArrayInit(10, sizeof(SQueryExplainRowInfo)); if (NULL == rows) { qError("taosArrayInit SQueryExplainRowInfo failed"); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } char *tbuf = taosMemoryMalloc(TSDB_EXPLAIN_RESULT_ROW_SIZE); if (NULL == tbuf) { qError("malloc size %d failed", TSDB_EXPLAIN_RESULT_ROW_SIZE); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ctx->mode = mode; @@ -212,6 +213,11 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = lastRowPhysiNode->scan.node.pChildren; break; } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { + STableCountScanPhysiNode *tableCountPhysiNode = (STableCountScanPhysiNode *)pNode; + pPhysiChildren = tableCountPhysiNode->scan.node.pChildren; + break; + } case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SGroupSortPhysiNode *groupSortPhysiNode = (SGroupSortPhysiNode *)pNode; pPhysiChildren = groupSortPhysiNode->node.pChildren; @@ -229,14 +235,14 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo } default: qError("not supported physical node type %d", pNode->type); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (pPhysiChildren) { *pChildren = nodesMakeList(); if (NULL == *pChildren) { qError("nodesMakeList failed"); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -254,7 +260,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S *pExecInfo = taosArrayInit(group->nodeNum, sizeof(SExplainExecInfo)); if (NULL == (*pExecInfo)) { qError("taosArrayInit %d explainExecInfo failed", group->nodeNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SExplainRsp *rsp = NULL; @@ -266,7 +272,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, group->nodeIdx++); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -275,7 +281,7 @@ int32_t qExplainGenerateResNodeExecInfo(SPhysiNode *pNode, SArray **pExecInfo, S rsp = taosArrayGet(group->nodeExecInfo, i); if (group->physiPlanExecIdx >= rsp->numOfPlans) { qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); @@ -291,13 +297,13 @@ int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplai if (NULL == pNode) { *pResNode = NULL; qError("physical node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExplainResNode *resNode = taosMemoryCalloc(1, sizeof(SExplainResNode)); if (NULL == resNode) { qError("calloc SPhysiNodeExplainRes failed"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = 0; @@ -372,7 +378,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t row.buf = taosMemoryMalloc(len); if (NULL == row.buf) { qError("taosMemoryMalloc %d failed", len); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(row.buf, tbuf, len); @@ -383,7 +389,7 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); taosMemoryFree(row.buf); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -401,7 +407,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SPhysiNode *pNode = pResNode->pNode; if (NULL == pNode) { qError("pyhsical node in explain res node is NULL"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } switch (pNode->type) { @@ -787,7 +793,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i SExplainGroup *group = taosHashGet(ctx->groupHash, &pExchNode->srcStartGroupId, sizeof(pExchNode->srcStartGroupId)); if (NULL == group) { qError("exchange src group %d not in groupHash", pExchNode->srcStartGroupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } nodeNum += group->nodeNum; @@ -1355,6 +1361,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { + STableCountScanPhysiNode *pLastRowNode = (STableCountScanPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_TABLE_COUNT_SCAN_FORMAT, + ('\0' != pLastRowNode->scan.tableName.tname[0] ? pLastRowNode->scan.tableName.tname : TSDB_INS_TABLE_TABLES)); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, LIST_LENGTH(pLastRowNode->scan.pScanCols)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + if (pLastRowNode->scan.pScanPseudoCols) { + EXPLAIN_ROW_APPEND(EXPLAIN_PSEUDO_COLUMNS_FORMAT, pLastRowNode->scan.pScanPseudoCols->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_OUTPUT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_COLUMNS_FORMAT, + nodesGetOutputNumFromSlotList(pLastRowNode->scan.node.pOutputDataBlockDesc->pSlots)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pLastRowNode->scan.node.pOutputDataBlockDesc->outputRowSize); + EXPLAIN_ROW_APPEND_LIMIT(pLastRowNode->scan.node.pLimit); + EXPLAIN_ROW_APPEND_SLIMIT(pLastRowNode->scan.node.pSlimit); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + if (pLastRowNode->scan.node.pConditions) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pLastRowNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + } + break; + } case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { SGroupSortPhysiNode *pSortNode = (SGroupSortPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_GROUP_SORT_FORMAT); @@ -1537,7 +1585,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } default: qError("not supported physical node type %d", pNode->type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1546,7 +1594,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i int32_t qExplainResNodeToRows(SExplainResNode *pResNode, SExplainCtx *ctx, int32_t level) { if (NULL == pResNode) { qError("explain res node is NULL"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t code = 0; @@ -1566,7 +1614,7 @@ int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, b SExplainGroup *group = taosHashGet(ctx->groupHash, &groupId, sizeof(groupId)); if (NULL == group) { qError("group %d not in groupHash", groupId); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } group->singleChannel = singleChannel; @@ -1588,7 +1636,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t rowNum = taosArrayGetSize(pCtx->rows); if (rowNum <= 0) { qError("empty explain res rows"); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } SSDataBlock *pBlock = createDataBlock(); @@ -1611,11 +1659,11 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); blockDataDestroy(pBlock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } rsp->completed = 1; - rsp->numOfRows = htonl(rowNum); + rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); @@ -1650,7 +1698,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { taosHashInit(EXPLAIN_MAX_GROUP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == groupHash) { qError("groupHash %d failed", EXPLAIN_MAX_GROUP_NUM); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QRY_ERR_JRET( @@ -1684,7 +1732,7 @@ int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { if (0 != taosHashPut(groupHash, &plan->id.groupId, sizeof(plan->id.groupId), &group, sizeof(group))) { qError("taosHashPut to explainGroupHash failed, taskIdx:%d", n); - QRY_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1752,7 +1800,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t if (NULL == group) { qError("group %d not in groupHash", groupId); tFreeSExplainRsp(pRspMsg); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosWLockLatch(&group->lock); @@ -1763,7 +1811,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QRY_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } group->physiPlanExecNum = pRspMsg->numOfPlans; @@ -1773,7 +1821,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } if (group->physiPlanExecNum != pRspMsg->numOfPlans) { @@ -1782,7 +1830,7 @@ int32_t qExplainUpdateExecInfo(SExplainCtx *pCtx, SExplainRsp *pRspMsg, int32_t tFreeSExplainRsp(pRspMsg); taosWUnLockLatch(&group->lock); - QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QRY_ERR_RET(TSDB_CODE_APP_ERROR); } taosArrayPush(group->nodeExecInfo, pRspMsg); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3dbcd8a07e68a98e0255049c22fb2b7925d46c46..3d3a08c8d04b6f36c4dccdafcc36619e8d6058f9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -46,7 +46,6 @@ extern "C" { typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) #define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) #define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) @@ -109,6 +108,7 @@ typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); typedef void (*__optr_close_fn_t)(void* param); typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id @@ -126,12 +126,14 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - const SSubmitReq* pReq; + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + // const SSubmitReq* pReq; + + SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -170,8 +172,9 @@ struct SExecTaskInfo { STaskCostInfo cost; int64_t owner; // if it is in execution int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query - int64_t version; // used for stream to record wal version + int64_t version; // used for stream to record wal version, why not move to sschemainfo SStreamTaskInfo streamInfo; SSchemaInfo schemaInfo; STableListInfo* pTableInfoList; // this is a table list @@ -181,7 +184,7 @@ struct SExecTaskInfo { SSubplan* pSubplan; struct SOperatorInfo* pRoot; SLocalFetch localFetch; - SArray* pResultBlockList;// result block list + SArray* pResultBlockList; // result block list STaskStopInfo stopInfo; }; @@ -198,6 +201,7 @@ typedef struct SOperatorFpSet { __optr_fn_t getNextFn; __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP __optr_close_fn_t closeFn; + __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator __optr_encode_fn_t encodeResultRow; __optr_decode_fn_t decodeResultRow; __optr_explain_fn_t getExplainFn; @@ -253,22 +257,22 @@ typedef struct SLimitInfo { } SLimitInfo; typedef struct SExchangeInfo { - SArray* pSources; - SArray* pSourceDataInfo; - tsem_t ready; - void* pTransporter; + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; // SArray, result block list, used to keep the multi-block that // passed by downstream operator - SArray* pResultBlockList; - SArray* pRecycledBlocks;// build a pool for small data block to avoid to repeatly create and then destroy. - SSDataBlock* pDummyBlock; // dummy block, not keep data - bool seqLoadData; // sequential load data or not, false by default - int32_t current; + SArray* pResultBlockList; + SArray* pRecycledBlocks; // build a pool for small data block to avoid to repeatly create and then destroy. + SSDataBlock* pDummyBlock; // dummy block, not keep data + bool seqLoadData; // sequential load data or not, false by default + int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; SLimitInfo limitInfo; - int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo + int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo } SExchangeInfo; typedef struct SScanInfo { @@ -303,9 +307,9 @@ typedef struct { } SAggOptrPushDownInfo; typedef struct STableMetaCacheInfo { - SLRUCache* pTableMetaEntryCache; // 100 by default - uint64_t metaFetch; - uint64_t cacheHit; + SLRUCache* pTableMetaEntryCache; // 100 by default + uint64_t metaFetch; + uint64_t cacheHit; } STableMetaCacheInfo; typedef struct STableScanBase { @@ -323,47 +327,47 @@ typedef struct STableScanBase { } STableScanBase; typedef struct STableScanInfo { - STableScanBase base; - SScanInfo scanInfo; - int32_t scanTimes; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; - int8_t scanMode; - int8_t assignBlockUid; - bool hasGroupByTag; + STableScanBase base; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + int32_t currentGroupId; + int32_t currentTable; + int8_t scanMode; + int8_t assignBlockUid; + bool hasGroupByTag; } STableScanInfo; typedef struct STableMergeScanInfo { - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond - STableScanBase base; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - SArray* sortSourceParams; - SLimitInfo limitInfo; - int64_t numOfRows; - SScanInfo scanInfo; - int32_t scanTimes; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - SSortExecInfo sortExecInfo; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* queryConds; // array of queryTableDataCond + STableScanBase base; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; + SLimitInfo limitInfo; + int64_t numOfRows; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + SSortExecInfo sortExecInfo; } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - SColMatchInfo matchInfo; - int32_t curPos; - SReadHandle readHandle; + SColumnInfo* pCols; + SSDataBlock* pRes; + SColMatchInfo matchInfo; + int32_t curPos; + SReadHandle readHandle; } STagScanInfo; typedef enum EStreamScanMode { @@ -482,6 +486,26 @@ typedef struct { SSnapContext* sContext; } SStreamRawScanInfo; +typedef struct STableCountScanSupp { + int16_t dbNameSlotId; + int16_t stbNameSlotId; + int16_t tbCountSlotId; + bool groupByDbName; + bool groupByStbName; + char dbNameFilter[TSDB_DB_NAME_LEN]; + char stbNameFilter[TSDB_TABLE_NAME_LEN]; +} STableCountScanSupp; + +typedef struct STableCountScanOperatorInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + + STableCountScanSupp supp; + + int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and/or stable_name +} STableCountScanOperatorInfo; + typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; @@ -649,27 +673,28 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain); -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +int32_t optrDummyOpenFn(SOperatorInfo* pOperator); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void setOperatorCompleted(SOperatorInfo* pOperator); void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, void* pInfo, SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); +int32_t optrDefaultBufFn(SOperatorInfo* pOperator); -void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); -void cleanupBasicInfo(SOptrBasicInfo* pInfo); +void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); +void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSupp(SExprSupp* pSup); -void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); +void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); void cleanupAggSup(SAggSupporter* pAggSup); -void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); +void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); @@ -684,7 +709,7 @@ void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pC int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); @@ -718,6 +743,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); @@ -780,18 +807,16 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t checkForQueryBuf(size_t numOfTables); -bool isTaskKilled(SExecTaskInfo* pTaskInfo); -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); -void doDestroyTask(SExecTaskInfo* pTaskInfo); -void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); +bool isTaskKilled(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); +void doDestroyTask(SExecTaskInfo* pTaskInfo); +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, char* sql, EOPTR_EXEC_MODEL model); int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t getMaximumIdleDurationSec(); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, @@ -807,8 +832,8 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); -uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); +void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index c432f3c01c18f333301201543144c043134a0b87..672bb09b14ecdba9e79c674d850100e8a07195ff 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -117,7 +117,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, NULL); + createOperatorFpSet(optrDummyOpenFn, doScanCache, NULL, destroyCacheScanOperator, optrDefaultBufFn, NULL); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index c7a248020472e229a2f786f8918e71fc8a36d765..eff7a5ef938379a1d6e6a078c302e3f91ca9f544 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -133,14 +133,14 @@ static int32_t getStatus(SDataDeleterHandle* pDeleter) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; - SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM); + SDataDeleterBuf* pBuf = taosAllocateQitem(sizeof(SDataDeleterBuf), DEF_QITEM, 0); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDeleter, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDeleter, pInput, pBuf); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d4248fc420462f63a35048bb0643623ec784afe5..c2fa438c808d42854794a56695d73ae192834247 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -126,14 +126,14 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; - SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM); + SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0); if (NULL == pBuf) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } if (!allocBuf(pDispatcher, pInput, pBuf)) { taosFreeQitem(pBuf); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } toDataCacheEntry(pDispatcher, pInput, pBuf); @@ -237,8 +237,8 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle) { SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle)); if (NULL == dispatcher) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fEndPut = endPut; @@ -254,8 +254,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD taosThreadMutexInit(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { taosMemoryFree(dispatcher); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pHandle = dispatcher; return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 09ca1d27b914c1ff888cd6e5c7990e6642343f4d..ca6149d42c90b21a7b1359348220d9611351dbf6 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -27,7 +27,7 @@ extern SDataSinkStat gDataSinkStat; typedef struct SSubmitRes { int64_t affectedRows; int32_t code; - SSubmitRsp* pRsp; + SSubmitRsp2* pRsp; } SSubmitRes; typedef struct SDataInserterHandle { @@ -58,22 +58,25 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.code = code; if (code == TSDB_CODE_SUCCESS) { - pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp)); + pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp); + code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); if (code) { - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } - if (pInserter->submitRes.pRsp->nBlocks > 0) { - for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + if (pInserter->submitRes.pRsp->affectedRows > 0) { + SArray* pCreateTbList = pInserter->submitRes.pRsp->aCreateTbRsp; + int32_t numOfTables = taosArrayGetSize(pCreateTbList); + + for (int32_t i = 0; i < numOfTables; ++i) { + SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (TSDB_CODE_SUCCESS != pRsp->code) { + code = pRsp->code; + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } @@ -83,25 +86,22 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); - - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + tDecoderClear(&coder); + taosMemoryFree(pInserter->submitRes.pRsp); } _return: - tsem_post(&pInserter->ready); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; } -static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) { +static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int32_t msgLen, void* pTransporter, SEpSet* pEpset) { // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { taosMemoryFreeClear(pMsg); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -111,7 +111,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); + pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; pMsgSendInfo->fp = inserterCallback; @@ -119,140 +119,233 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo); } -int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { - const SArray* pBlocks = pInserter->pDataBlocks; - const STSchema* pTSchema = pInserter->pSchema; - int64_t uid = pInserter->pNode->tableId; - int64_t suid = pInserter->pNode->stableId; - int32_t vgId = pInserter->pNode->vgId; - bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); +static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SMsgHead); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SMsgHead*)pBuf)->vgId = htonl(vgId); + ((SMsgHead*)pBuf)->contLen = htonl(len); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SMsgHead)), len - sizeof(SMsgHead)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); + } - SSubmitReq* ret = NULL; - int32_t sz = taosArrayGetSize(pBlocks); + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; +} - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - cap += sizeof(SSubmitBlk) + rows * maxLen; +int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols); + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } - // assign data - // TODO - ret = taosMemoryCalloc(1, cap); - ret->header.vgId = htonl(vgId); - ret->version = htonl(pTSchema->version); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + SSubmitTbData tbData = {0}; + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - blkHead->uid = htobe64(uid); - blkHead->schemaLen = htonl(0); - - int32_t rows = 0; - int32_t dataLen = 0; - STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } - ignoreRow = false; - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = NULL; - int16_t colIdx = k; - if (!fullCol) { - int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); - if (NULL == slotId) { - continue; - } + int64_t lastTs = TSKEY_MIN; + bool ignoreRow = false; + bool disorderTs = false; - colIdx = *slotId; - } + for (int32_t j = 0; j < rows; ++j) { // iterate by row + taosArrayClear(pVals); - pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx); - if (pColData->info.type != pColumn->type) { - qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type); - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_APP_ERROR; + int32_t offset = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; ++k) { // iterate by column + int16_t colIdx = k; + const STColumn* pCol = &pTSchema->columns[k]; + if (!fullCol) { + int16_t* slotId = taosHashGet(pInserter->pCols, &pCol->colId, sizeof(pCol->colId)); + if (NULL == slotId) { + continue; } - if (colDataIsNull_s(pColData, j)) { - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - ignoreRow = true; - break; + colIdx = *slotId; + } + + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } - - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - if (*(int64_t*)data == lastTs) { - ignoreRow = true; - break; + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); } else { - lastTs = *(int64_t*)data; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + if (*(int64_t*)var == lastTs) { + ignoreRow = true; + } else if (*(int64_t*)var < lastTs) { + disorderTs = true; + } else { + lastTs = *(int64_t*)var; + } + } + + SValue sv; + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - if (!fullCol) { - rb.hasNone = true; + break; } - tdSRowEnd(&rb); if (ignoreRow) { - continue; + break; } - - rows++; - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; } - blkHead->dataLen = htonl(dataLen); - blkHead->numOfRows = htonl(rows); + if (ignoreRow) { + ignoreRow = false; + continue; + } + + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); + } - ret->length += sizeof(SSubmitBlk) + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen); + if (disorderTs) { + tRowSort(tbData.aRowP); + if ((terrno = tRowMerge(tbData.aRowP, (STSchema*)pTSchema, 0)) != 0) { + goto _end; + } } - ret->length = htonl(ret->length); + taosArrayPush(pReq->aSubmitTbData, &tbData); - *pReq = ret; +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + return TSDB_CODE_FAILED; + } + *ppReq = pReq; return TSDB_CODE_SUCCESS; } + +int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32_t* msgLen) { + const SArray* pBlocks = pInserter->pDataBlocks; + const STSchema* pTSchema = pInserter->pSchema; + int64_t uid = pInserter->pNode->tableId; + int64_t suid = pInserter->pNode->stableId; + int32_t vgId = pInserter->pNode->vgId; + int32_t sz = taosArrayGetSize(pBlocks); + int32_t code = 0; + SSubmitReq2 *pReq = NULL; + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + + code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); + if (code) { + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + return code; + } + } + + code = submitReqToMsg(vgId, pReq, pMsg, msgLen); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + + return code; +} + static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; taosArrayPush(pInserter->pDataBlocks, &pInput->pData); - SSubmitReq* pMsg = NULL; - int32_t code = dataBlockToSubmit(pInserter, &pMsg); + void* pMsg = NULL; + int32_t msgLen = 0; + int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); if (code) { return code; } taosArrayClear(pInserter->pDataBlocks); - code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); + code = sendSubmitRequest(pInserter, pMsg, msgLen, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); if (code) { return code; } @@ -304,8 +397,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SQueryInserterNode* pInserterNode = (SQueryInserterNode*)pDataSink; @@ -342,8 +435,8 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat if (NULL == inserter->pDataBlocks) { destroyDataSinker((SDataSinkHandle*)inserter); taosMemoryFree(inserter); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 53660d88e1bd819e0adbd5f3fe556f3eca7b63f7..4103ca82dcc7b122f925207717ea15698bf7b904 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -115,14 +115,14 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn if (pRsp->completed == 1) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d index:%d completed, blocks:%d, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + " execId:%d index:%d completed, blocks:%d, numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", total:%.2f Kb, try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRsp->numOfBlocks, pRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0, i + 1, totalSources); } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 - " execId:%d blocks:%d, numOfRows:%d, totalRows:%" PRIu64 ", total:%.2f Kb", + " execId:%d blocks:%d, numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", total:%.2f Kb", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRsp->numOfBlocks, pRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize / 1024.0); } @@ -198,7 +198,7 @@ static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { } } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { +static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -307,7 +307,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); pOperator->fpSet = - createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, destroyExchangeOperatorInfo, NULL); + createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _error: @@ -367,14 +367,14 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { pSourceDataInfo->pRsp = pMsg->pData; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->numOfRows = htobe64(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); ASSERT(pRsp != NULL); - qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%d, %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, + qDebug("%s fetch rsp received, index:%d, blocks:%d, rows:%" PRId64 ", %p", pSourceDataInfo->taskId, index, pRsp->numOfBlocks, pRsp->numOfRows, pExchangeInfo); } else { taosMemoryFree(pMsg->pData); @@ -424,20 +424,20 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } void* msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pWrapper); taosMemoryFree(msg); return pTaskInfo->code; @@ -453,7 +453,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return pTaskInfo->code; } @@ -472,7 +472,7 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas return TSDB_CODE_SUCCESS; } -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, SOperatorInfo* pOperator) { pInfo->totalRows += numOfRows; pInfo->totalSize += dataLen; @@ -510,6 +510,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo blockDataEnsureCapacity(pRes, pBlock->info.rows); // data from mnode + pRes->info.dataLoad = 1; pRes->info.rows = pBlock->info.rows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); blockDataDestroy(pBlock); @@ -570,13 +571,10 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { pOperator->status = OP_RES_TO_RETURN; pOperator->cost.openCost = taosGetTimestampUs() - startTs; - - tsem_wait(&pExchangeInfo->ready); if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, pTaskInfo->code); } - tsem_post(&pExchangeInfo->ready); return TSDB_CODE_SUCCESS; } @@ -655,7 +653,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; if (pRsp->completed == 1) { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, @@ -664,7 +662,7 @@ int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%" PRId64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRetrieveRsp->numOfRows, pLoadInfo->totalRows, pLoadInfo->totalSize); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 8a85885d989303d1bbae422155ad0b3be82d9ab1..fc3cfbd0f65bc7668679fc23aa8466a11ee9ac81 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -129,6 +129,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void* pData = NULL; pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + // todo avoid repeated malloc memory size_t keyLen = 0; int32_t iter = 0; while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { @@ -438,13 +439,14 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } } + if (suid != 0) { + removeInvalidTable(uidList, tags); + } int32_t rows = taosArrayGetSize(uidList); if (rows == 0) { goto end; } - // int64_t stt1 = taosGetTimestampUs(); - // qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { @@ -452,7 +454,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* goto end; } - // int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { int64_t* uid = taosArrayGet(uidList, i); for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) { @@ -467,7 +468,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* #endif } else { void* tag = taosHashGet(tags, uid, sizeof(int64_t)); - ASSERT(tag); + if (tag == NULL) { + continue; + } STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); @@ -923,14 +926,14 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* return -1; } - SArray* pTbList = getTableNameList(pList); - int32_t numOfTables = taosArrayGetSize(pTbList); - SHashObj *uHash = NULL; + SArray* pTbList = getTableNameList(pList); + int32_t numOfTables = taosArrayGetSize(pTbList); + SHashObj* uHash = NULL; size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids if (listlen > 0) { uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < listlen; i++) { - int64_t *uid = taosArrayGet(list, i); + int64_t* uid = taosArrayGet(list, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } } @@ -1241,6 +1244,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod } } + // set the output flag for each column in SColMatchInfo, according to the *numOfOutputCols = 0; int32_t num = LIST_LENGTH(pOutputNodeList->pSlots); for (int32_t i = 0; i < num; ++i) { @@ -1348,6 +1352,7 @@ void createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId) { pExprNode->_function.functionId = pFuncNode->funcId; pExprNode->_function.pFunctNode = pFuncNode; + pExprNode->_function.functionType = pFuncNode->funcType; tstrncpy(pExprNode->_function.functionName, pFuncNode->functionName, tListLen(pExprNode->_function.functionName)); @@ -1455,7 +1460,7 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu SqlFunctionCtx* p = NULL; SqlFunctionCtx** pValCtx = taosMemoryCalloc(numOfOutput, POINTER_BYTES); if (pValCtx == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1536,8 +1541,6 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->start.key = INT64_MIN; pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; - pCtx->isStream = false; - pCtx->param = pFunct->pParam; pCtx->saveHandle.currentPage = -1; } @@ -1601,20 +1604,22 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); + pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); return terrno; } - // pCond->twindow = pTableScanNode->scanRange; // TODO: get it from stable scan node pCond->twindows = pTableScanNode->scanRange; pCond->suid = pTableScanNode->scan.suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { @@ -1627,6 +1632,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->colList[j].type = pColNode->node.resType.type; pCond->colList[j].bytes = pColNode->node.resType.bytes; pCond->colList[j].colId = pColNode->colId; + + pCond->pSlotList[j] = pNode->slotId; j += 1; } @@ -1634,7 +1641,10 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi return TSDB_CODE_SUCCESS; } -void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFreeClear(pCond->colList); } +void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); +} int32_t convertFillType(int32_t mode) { int32_t type = TSDB_FILL_NONE; @@ -1964,7 +1974,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, - struct SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo) { int64_t st = taosGetTimestampUs(); const char* idStr = GET_TASKID(pTaskInfo); @@ -2011,4 +2021,4 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag) { char* pBuf = NULL; qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); taosMemoryFree(pBuf); -} \ No newline at end of file +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ebd1afa85539eb9f2e660c3707b16a2c97e32240..3e0c5a990148e6e92d95ef24c1152da234645a16 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -35,12 +35,12 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetSMABlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -51,8 +51,8 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -61,7 +61,10 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } @@ -76,12 +79,12 @@ static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamOpOpen(pOperator->pDownstream[0], id); @@ -95,12 +98,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query qError("join not supported for stream block scan, %s" PRIx64, id); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pOperator->status = OP_NOT_OPENED; return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); @@ -115,18 +118,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(numOfBlocks == 1); - taosArrayPush(pInfo->pBlockLists, &input); + taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { @@ -139,7 +145,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; @@ -156,7 +162,7 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -177,7 +183,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pBlocks == NULL || numOfBlocks == 0) { @@ -207,7 +213,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* n } setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->cost.created = taosGetTimestampUs(); pTaskInfo->execModel = OPTR_EXEC_MODEL_QUEUE; pTaskInfo->pRoot = createRawScanOperatorInfo(readers, pTaskInfo); if (NULL == pTaskInfo->pRoot) { @@ -503,7 +509,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -539,7 +545,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo taosArrayPush(pTaskInfo->pResultBlockList, &p1); p = p1; } else { - p = *(SSDataBlock**) taosArrayGet(pTaskInfo->pResultBlockList, blockIndex); + p = *(SSDataBlock**)taosArrayGet(pTaskInfo->pResultBlockList, blockIndex); copyDataBlock(p, pRes); } @@ -574,9 +580,9 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - SArray* pList = pTaskInfo->pResultBlockList; - size_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { + SArray* pList = pTaskInfo->pResultBlockList; + size_t num = taosArrayGetSize(pList); + for (int32_t i = 0; i < num; ++i) { SSDataBlock** p = taosArrayGet(pTaskInfo->pResultBlockList, i); blockDataDestroy(*p); } @@ -597,7 +603,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds) { } if (pTaskInfo->cost.start == 0) { - pTaskInfo->cost.start = taosGetTimestampMs(); + pTaskInfo->cost.start = taosGetTimestampUs(); } if (isTaskKilled(pTaskInfo)) { @@ -706,15 +712,20 @@ int32_t qAsyncKillTask(qTaskInfo_t qinfo, int32_t rspCode) { static void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; + int64_t idleTime = pSummary->start - pSummary->created; SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "%s :cost summary: idle:%.2f ms, elapsed time:%.2f ms, extract tableList:%.2f ms, " + "createGroupIdMap:%.2f ms, total blocks:%d, " "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, - pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, - pRecorder->totalCheckedRows); + GET_TASKID(pTaskInfo), idleTime / 1000.0, pSummary->elapsedTime / 1000.0, pSummary->extractListTime, + pSummary->groupIdMapTime, pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, + pRecorder->totalRows, pRecorder->totalCheckedRows); + } else { + qDebug("%s :cost summary: idle in queue:%.2f ms, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), idleTime / 1000.0, + pSummary->elapsedTime / 1000.0); } } @@ -742,11 +753,11 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { } int32_t nOptrWithVal = 0; -// int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); -// if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { -// taosMemoryFreeClear(*pOutput); -// *len = 0; -// } + // int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); + // if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { + // taosMemoryFreeClear(*pOutput); + // *len = 0; + // } return 0; } @@ -758,7 +769,7 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le } return 0; -// return decodeOperator(pTaskInfo->pRoot, pInput, len); + // return decodeOperator(pTaskInfo->pRoot, pInput, len); } int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { @@ -971,31 +982,48 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s pCond->order = TSDB_ORDER_ASC; pCond->numOfCols = pMtInfo->schema->nCols; pCond->colList = taosMemoryCalloc(pCond->numOfCols, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t) * pCond->numOfCols); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + taosMemoryFreeClear(pCond->colList); + taosMemoryFreeClear(pCond->pSlotList); + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->twindows = TSWINDOW_INITIALIZER; pCond->suid = pMtInfo->suid; pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { - pCond->colList[i].type = pMtInfo->schema->pSchema[i].type; - pCond->colList[i].bytes = pMtInfo->schema->pSchema[i].bytes; - pCond->colList[i].colId = pMtInfo->schema->pSchema[i].colId; + SColumnInfo* pColInfo = &pCond->colList[i]; + pColInfo->type = pMtInfo->schema->pSchema[i].type; + pColInfo->bytes = pMtInfo->schema->pSchema[i].bytes; + pColInfo->colId = pMtInfo->schema->pSchema[i].colId; + + pCond->pSlotList[i] = i; } return TSDB_CODE_SUCCESS; } -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq) { +#if 0 +int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); ASSERT(pTaskInfo->streamInfo.pReq == NULL); pTaskInfo->streamInfo.pReq = pReq; + pTaskInfo->streamInfo.scanVer = scanVer; + return 0; +} +#endif + +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + ASSERT(pTaskInfo->streamInfo.submit.msgStr == NULL); + pTaskInfo->streamInfo.submit = submit; return 0; } @@ -1078,7 +1106,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || pTableScanInfo->base.dataReader == NULL) { ASSERT(0); } @@ -1130,7 +1158,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); ASSERT(size == 1); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index dd527058ceee4c32672fba04fd2983ba13d42ea9..1c80eff68547ebe9eac89aa7ff8098bdee0b45d4 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -85,8 +85,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; } - static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void releaseQueryBuf(size_t numOfTables); @@ -106,7 +104,7 @@ void setOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; ASSERT(pOperator->pTaskInfo != NULL); - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start * 1000) / 1000.0; + pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } @@ -120,19 +118,21 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b pOperator->pTaskInfo = pTaskInfo; } -int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { +int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { OPTR_SET_OPENED(pOperator); pOperator->cost.openCost = 0; return TSDB_CODE_SUCCESS; } SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_explain_fn_t explain) { + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, + __optr_explain_fn_t explain) { SOperatorFpSet fpSet = { ._openFn = openFn, .getNextFn = nextFn, .cleanupFn = cleanup, .closeFn = closeFn, + .reqBufFn = reqBufFn, .getExplainFn = explain, }; @@ -675,7 +675,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } else if (pQueryAttr->stableQuery && (!pQueryAttr->tsCompQuery) && (!pQueryAttr->diffQuery)) { // stable aggregate, not interval aggregate or normal column aggregate @@ -726,7 +726,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_OUT_OF_MEMORY); } } } @@ -939,10 +939,10 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u } static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, - const int32_t* rowCellOffset) { + const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { - struct SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowCellOffset); + SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (!isRowEntryInitialized(pResInfo)) { continue; } @@ -1080,7 +1080,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.id.groupId); - + pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); return 0; } @@ -1145,45 +1145,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -// void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// -// if (pQueryAttr->limit.offset <= 0 || pQueryAttr->numOfFilterCols > 0) { -// return; -// } -// -// pQueryAttr->pos = 0; -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; -// -// SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pTsdbReadHandle)) { -// if (isTaskKilled(pRuntimeEnv->qinfo)) { -// T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); -// } -// -// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); -// -// if (pQueryAttr->limit.offset > blockInfo.rows) { -// pQueryAttr->limit.offset -= blockInfo.rows; -// pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; -// pTableQueryInfo->lastKey += step; -// -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, -// pQuery->limit.offset); -// } else { // find the appropriated start position in current block -// updateOffsetVal(pRuntimeEnv, &blockInfo); -// break; -// } -// } -// -// if (terrno != TSDB_CODE_SUCCESS) { -// T_LONG_JMP(pRuntimeEnv->env, terrno); -// } -// } - // static TSKEY doSkipIntervalProcess(STaskRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, // STableQueryInfo* pTableQueryInfo) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1370,7 +1331,8 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan int32_t type = pOperator->operatorType; if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE || type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN) { + type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { *order = TSDB_ORDER_ASC; *scanFlag = MAIN_SCAN; return TSDB_CODE_SUCCESS; @@ -1407,11 +1369,11 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; bool hasCountFunc = false; + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - if ((strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "count") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "hyperloglog") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_partial") == 0) || - (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_hyperloglog_merge") == 0)) { + const char* pName = pCtx[i].pExpr->pExpr->_function.functionName; + if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) || + (strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) { hasCountFunc = true; break; } @@ -1464,7 +1426,6 @@ static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppB *ppBlock = NULL; } - // this is a blocking operator static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { if (OPTR_IS_OPENED(pOperator)) { @@ -1615,6 +1576,16 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { taosMemoryFreeClear(pOperator); } +// each operator should be set their own function to return total cost buffer +int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { + if (pOperator->blocking) { + ASSERT(0); + return 0; + } else { + return 0; + } +} + int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { *defaultPgsz = 4096; while (*defaultPgsz < rowSize * 4) { @@ -1640,7 +1611,7 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n pAggSup->currentPageId = -1; pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); - pAggSup->pResultRowHashTable = tSimpleHashInit(10, hashFn); + pAggSup->pResultRowHashTable = tSimpleHashInit(100, hashFn); if (pAggSup->keyBuf == NULL || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1795,7 +1766,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pTableScanInfo = downstream->info; @@ -1856,7 +1827,6 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->schemaInfo.dbname = strdup(dbFName); - pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; pTaskInfo->execModel = model; pTaskInfo->pTableInfoList = tableListCreate(); @@ -2062,6 +2032,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) { + STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; + pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; @@ -2224,12 +2197,12 @@ static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanI if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { qError("failed to find stream scan operator"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pOperator->numOfDownstream > 1) { qError("join not supported for stream block scan"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); } else { @@ -2247,13 +2220,13 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return 0; } else { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } } else { if (LIST_LENGTH(pNode->pChildren) != 1) { ASSERT(0); - terrno = TSDB_CODE_QRY_APP_ERROR; + terrno = TSDB_CODE_APP_ERROR; return -1; } SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); @@ -2281,7 +2254,7 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa if (pTableScanInfo->dataReader == NULL) { ASSERT(0); qError("failed to create data reader"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } // TODO: set uid and ts to data reader return 0; @@ -2360,6 +2333,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } + (*pTaskInfo)->cost.created = taosGetTimestampUs(); return TSDB_CODE_SUCCESS; _complete: @@ -2455,7 +2429,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInf code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFreeClear(*pRes); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -2472,7 +2446,7 @@ int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResul int32_t size = pAggSup->resultRowSize; if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; ASSERT(*pResult); @@ -2572,6 +2546,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat pBlock->info.rows += pRow->numOfRows; releaseOutputBuf(pState, &key, pRow); } + pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } @@ -2661,6 +2636,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta } } + pBlock->info.dataLoad = 1; pBlock->info.rows += pRow->numOfRows; // saveSessionDiscBuf(pState, pKey, pVal, size); releaseOutputBuf(pState, NULL, pRow); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 5ed26f627a26736d21e478ea79fec0543b45df27..8d5af64777172adf3e24bf3edebc8c47438b7476 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -381,7 +381,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* setOperatorInfo(pOperator, "FillOperator", QUERY_NODE_PHYSICAL_PLAN_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = pInfo->numOfExpr; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doFill, NULL, destroyFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -1478,7 +1478,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi pInfo->srcRowIndex = 0; setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4601175561e3da53408b9afa2c9818e1baf004c4..59266dad7eef4040f79cb3383974453eb4727324 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -310,11 +310,12 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = j - num; - applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs); + applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, + pOperator->exprSupp.numOfExprs); // assign the group keys or user input constant values if required doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex); @@ -327,11 +328,12 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } int32_t rowIndex = pBlock->info.rows - num; - applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs); + applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, + pOperator->exprSupp.numOfExprs); doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex); } } @@ -469,8 +471,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* initResultRowInfo(&pInfo->binfo.resultRowInfo); setOperatorInfo(pOperator, "GroupbyAggOperator", 0, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, hashGroupbyAggregate, NULL, destroyGroupOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -696,6 +698,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->pageIndex += 1; releaseBufPage(pInfo->pBuf, page); + pInfo->binfo.pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->binfo.pRes, 0); pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; @@ -776,6 +779,12 @@ static void destroyPartitionOperatorInfo(void* param) { taosArrayDestroy(pInfo->pGroupColVals); taosMemoryFree(pInfo->keyBuf); + + int32_t size = taosArrayGetSize(pInfo->sortedGroupArray); + for (int32_t i = 0; i < size; i++) { + SDataGroupInfo* pGp = taosArrayGet(pInfo->sortedGroupArray, i); + taosArrayDestroy(pGp->pPageList); + } taosArrayDestroy(pInfo->sortedGroupArray); void* pGroupIter = taosHashIterate(pInfo->pGroupSet, NULL); @@ -850,7 +859,8 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, hashPartition, NULL, destroyPartitionOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -951,6 +961,8 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { } taosArrayDestroy(pParInfo->rowIds); pParInfo->rowIds = NULL; + pDest->info.dataLoad = 1; + blockDataUpdateTsWindow(pDest, pInfo->tsColIndex); pDest->info.id.groupId = pParInfo->groupId; pOperator->resultInfo.totalRows += pDest->info.rows; @@ -1141,8 +1153,8 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, destroyStreamPartitionOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamHashPartition, NULL, + destroyStreamPartitionOperatorInfo, optrDefaultBufFn, NULL); initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index e7cce39dfdfd0d9152e326d4b0c66a510dad5458..8a097a23ceeee705907cb8460a5b34f43985effa 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -87,11 +87,11 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t } int32_t numOfCols = 0; - SSDataBlock* pResBlock = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); + pInfo->pRes = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc); + SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols); initResultSizeInfo(&pOperator->resultInfo, 4096); - - pInfo->pRes = pResBlock; + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); setOperatorInfo(pOperator, "MergeJoinOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.pExprInfo = pExprInfo; @@ -136,7 +136,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t pInfo->inputOrder = TSDB_ORDER_DESC; } - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeJoin, NULL, destroyMergeJoinOperator, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, pDownstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -401,6 +401,7 @@ static void doMergeJoinImpl(struct SOperatorInfo* pOperator, SSDataBlock* pRes) // the pDataBlock are always the same one, no need to call this again pRes->info.rows = nrows; + pRes->info.dataLoad = 1; if (pRes->info.rows >= pOperator->resultInfo.threshold) { break; } @@ -412,7 +413,7 @@ SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator) { SSDataBlock* pRes = pJoinInfo->pRes; blockDataCleanup(pRes); - blockDataEnsureCapacity(pRes, 4096); + while (true) { int32_t numOfRowsBefore = pRes->info.rows; doMergeJoinImpl(pOperator, pRes); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index da77facb2184363b14e65053aca379afd56589fb..221ba0e8d6e90a4d9ecd15bdb46071eb72100731 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "filter.h" #include "executorimpl.h" +#include "filter.h" #include "functionMgt.h" typedef struct SProjectOperatorInfo { @@ -90,7 +90,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->binfo.pRes = pResBlock; pInfo->pFinalRes = createOneDataBlock(pResBlock, false); - pInfo->mergeDataBlocks = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM)? false:pProjPhyNode->mergeDataBlock; + pInfo->mergeDataBlocks = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? false : pProjPhyNode->mergeDataBlock; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -117,9 +117,10 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pOperator->exprSupp.pCtx, numOfCols); - setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, - destroyProjectOperatorInfo, NULL); + setOperatorInfo(pOperator, "ProjectOperator", QUERY_NODE_PHYSICAL_PLAN_PROJECT, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doProjectOperation, NULL, destroyProjectOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -220,7 +221,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.pReq) { + if (pTaskInfo->streamInfo.submit.msgStr) { pOperator->status = OP_OPENED; } @@ -414,8 +415,10 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pInfo->binfo.pRes = pResBlock; pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pSup->pCtx, numOfExpr); - setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, NULL); + setOperatorInfo(pOperator, "IndefinitOperator", QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, false, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doApplyIndefinitFunction, NULL, destroyIndefinitOperatorInfo, + optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -654,6 +657,7 @@ static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, S int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); + pResult->info.dataLoad = 1; if (pSrcBlock == NULL) { for (int32_t k = 0; k < numOfOutput; ++k) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cf0e7b532f1ce8ad0c31bdb2f055dd63a32fc13f..76c07a9611f189aa3d6c6a6484b84cdb50fea375 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -30,7 +30,6 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -111,9 +110,9 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn if (order == TSDB_ORDER_ASC) { w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.skey); - assert(w.ekey >= pBlockInfo->window.skey); + ASSERT(w.ekey >= pBlockInfo->window.skey); - if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) { + if (w.ekey < pBlockInfo->window.ekey) { return true; } @@ -123,16 +122,16 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn break; } - assert(w.ekey > pBlockInfo->window.ekey); + ASSERT(w.ekey > pBlockInfo->window.ekey); if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) { return true; } } } else { w = getAlignQueryTimeWindow(pInterval, pInterval->precision, pBlockInfo->window.ekey); - assert(w.skey <= pBlockInfo->window.ekey); + ASSERT(w.skey <= pBlockInfo->window.ekey); - if (TMAX(w.skey, pBlockInfo->window.skey) <= TMIN(w.ekey, pBlockInfo->window.ekey)) { + if (w.skey > pBlockInfo->window.skey) { return true; } @@ -224,10 +223,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA } static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - bool allColumnsHaveAgg = true; - SColumnDataAgg** pColAgg = NULL; - - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + bool allColumnsHaveAgg = true; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -236,6 +233,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, return false; } +#if 0 // if (allColumnsHaveAgg == true) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -256,6 +254,7 @@ static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, pBlock->pBlockAgg[pColMatchInfo->dstSlotId] = pColAgg[i]; } +#endif return true; } @@ -285,7 +284,7 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo if (pLimit->offset > 0 && pLimitInfo->remainOffset > 0) { if (pLimitInfo->remainOffset >= pBlock->info.rows) { pLimitInfo->remainOffset -= pBlock->info.rows; - pBlock->info.rows = 0; + blockDataEmpty(pBlock); qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id); } else { blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); @@ -385,12 +384,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; - SArray* pCols = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); - if (pCols == NULL) { + SSDataBlock* p = tsdbRetrieveDataBlock(pTableScanInfo->dataReader, NULL); + if (p == NULL) { return terrno; } - relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); + ASSERT(p == pBlock); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value @@ -487,10 +486,11 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { - qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, - tstrerror(terrno), idStr); + qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", + pBlock->info.id.uid, tstrerror(terrno), idStr); } else { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr); + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), + idStr); } metaReaderClear(&mr); return terrno; @@ -638,16 +638,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - blockDataCleanup(pBlock); - SDataBlockInfo* pBInfo = &pBlock->info; - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - blockDataEnsureCapacity(pBlock, rows); // todo remove it latter - pBInfo->rows = rows; - - ASSERT(pBInfo->id.uid != 0); + ASSERT(pBlock->info.id.uid != 0); pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); uint32_t status = 0; @@ -777,7 +768,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { tableListGetGroupList(pTaskInfo->pTableInfoList, pInfo->currentGroupId, &pList, &num); ASSERT(pInfo->base.dataReader == NULL); - int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, + int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -879,11 +870,11 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->base.readHandle = *readHandle; + pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; + pInfo->sample.sampleRatio = pTableScanNode->ratio; pInfo->sample.seed = taosGetTimestampSec(); - pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; - initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); @@ -908,8 +899,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, - getTableScannerExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, + optrDefaultBufFn, getTableScannerExecInfo); // for non-blocking operator, the open cost is always 0 pOperator->cost.openCost = 0; @@ -934,7 +925,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImpl, NULL, NULL, optrDefaultBufFn, NULL); return pOperator; } @@ -994,32 +985,19 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo; SSDataBlock* pBlock = pTableScanInfo->pResBlock; - blockDataCleanup(pBlock); - STsdbReader* pReader = NULL; - int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, (STsdbReader**)&pReader, - GET_TASKID(pTaskInfo)); + int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, + (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); return NULL; } - bool hasBlock = tsdbNextDataBlock(pReader); - if (hasBlock) { - SDataBlockInfo* pBInfo = &pBlock->info; - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window); - - SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - - relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true); - doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows); - - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid); + if (tsdbNextDataBlock(pReader)) { + /*SSDataBlock* p = */ tsdbRetrieveDataBlock(pReader, NULL); + doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows); + pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); } tsdbReaderClose(pReader); @@ -1240,27 +1218,56 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS if (rows == 0) { return TSDB_CODE_SUCCESS; } - int32_t code = blockDataEnsureCapacity(pDestBlock, rows); - if (code != TSDB_CODE_SUCCESS) { - return code; - } SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); - uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; + + uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; - TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + int64_t version = pSrcBlock->info.version - 1; + + if (pInfo->partitionSup.needCalc && srcStartTsCol[0] != srcEndTsCol[0]) { + uint64_t srcUid = srcUidData[0]; + TSKEY startTs = srcStartTsCol[0]; + TSKEY endTs = srcEndTsCol[0]; + SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, startTs, endTs, version); + printDataBlock(pPreRes, "pre res"); + blockDataCleanup(pSrcBlock); + int32_t code = blockDataEnsureCapacity(pSrcBlock, pPreRes->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pPreRes->pDataBlock, pInfo->primaryTsIndex); + rows = pPreRes->info.rows; + + for (int32_t i = 0; i < rows; i++) { + uint64_t groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, i); + appendOneRowToStreamSpecialBlock(pSrcBlock, ((TSKEY*)pTsCol->pData) + i, ((TSKEY*)pTsCol->pData) + i, &srcUid, + &groupId, NULL); + } + printDataBlock(pSrcBlock, "new delete"); + } + uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; + srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; + srcUidData = (uint64_t*)pSrcUidCol->pData; + + int32_t code = blockDataEnsureCapacity(pDestBlock, rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { uint64_t srcUid = srcUidData[i]; uint64_t groupId = srcGp[i]; @@ -1335,6 +1342,7 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, } pDestBlock->info.type = STREAM_CLEAR; pDestBlock->info.version = pSrcBlock->info.version; + pDestBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pDestBlock, 0); return code; } @@ -1436,13 +1444,14 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock NULL); if (closedWin && pInfo->partitionSup.needCalc) { gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId); - appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId, - NULL); + appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, + &gpId, NULL); } } } if (out && pInfo->pUpdateDataRes->info.rows > 0) { pInfo->pUpdateDataRes->info.version = pBlock->info.version; + pInfo->pUpdateDataRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0); pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR; } @@ -1505,6 +1514,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); } + pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); @@ -1518,14 +1528,18 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("queue scan called"); - if (pTaskInfo->streamInfo.pReq != NULL) { - if (pInfo->tqReader->pMsg == NULL) { - pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq; - const SSubmitReq* pSubmit = pInfo->tqReader->pMsg; - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { - qError("submit msg messed up when initing stream submit block %p", pSubmit); - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + if (pTaskInfo->streamInfo.submit.msgStr != NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { + /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ + + /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + /*void* msgStr = pTaskInfo->streamInfo.*/ + SPackedData submit = pTaskInfo->streamInfo.submit; + if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; ASSERT(0); } } @@ -1533,10 +1547,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1549,8 +1563,9 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } @@ -1675,13 +1690,6 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]); colDataAppend(pGpCol, i, (const char*)&groupId, false); } - } else { - // SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uidCol[i], startTsCol, ts, maxVersion); - // if (!pPreRes || pPreRes->info.rows == 0) { - // return 0; - // } - // ASSERT(pPreRes->info.rows == 1); - // return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); } } @@ -1786,13 +1794,15 @@ FETCH_NEXT_BLOCK: } int32_t current = pInfo->validBlockIndex++; - SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + SSDataBlock* pBlock = pPacked->pDataBlock; if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.ekey = INT64_MAX; + pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); switch (pBlock->info.type) { case STREAM_NORMAL: @@ -1915,7 +1925,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->pMsg == NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -1923,22 +1933,22 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; - SSubmitReq* pSubmit = taosArrayGetP(pInfo->pBlockLists, current); - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { + int32_t current = pInfo->validBlockIndex++; + SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); - pInfo->tqReader->pMsg = NULL; continue; } } blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1970,6 +1980,7 @@ FETCH_NEXT_BLOCK: } doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { @@ -1979,7 +1990,6 @@ FETCH_NEXT_BLOCK: if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } else { - pInfo->tqReader->pMsg = NULL; continue; } /*blockDataCleanup(pInfo->pRes);*/ @@ -2028,20 +2038,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap doRawScan called"); if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { - SSDataBlock* pBlock = &pInfo->pRes; - if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { longjmp(pTaskInfo->env, pTaskInfo->code); } - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - pBlock->info.rows = rows; - - SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); - pBlock->pDataBlock = pCols; - if (pCols == NULL) { + SSDataBlock* pBlock = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); + if (pBlock == NULL) { longjmp(pTaskInfo->env, terrno); } @@ -2168,7 +2171,7 @@ SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pT setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(NULL, doRawScan, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; _end: @@ -2212,7 +2215,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -2267,7 +2270,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2287,7 +2290,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, &pTSInfo->base.dataReader, NULL); + code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, + &pTSInfo->base.dataReader, NULL); if (code != 0) { terrno = code; destroyTableScanOperatorInfo(pTableScanOp); @@ -2357,7 +2361,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); __optr_fn_t nextFn = pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM ? doStreamScan : doQueueScan; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2462,7 +2467,8 @@ static void destroyTagScanOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2493,7 +2499,8 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi initResultSizeInfo(&pOperator->resultInfo, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doTagScan, NULL, destroyTagScanOperatorInfo, optrDefaultBufFn, NULL); return pOperator; @@ -2511,17 +2518,15 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t readIdx = source->readerIdx; SSDataBlock* pBlock = source->inputBlock; - STableMergeScanInfo* pTableScanInfo = pOperator->info; - - SQueryTableDataCond* pQueryCond = taosArrayGet(pTableScanInfo->queryConds, readIdx); - blockDataCleanup(pBlock); - int64_t st = taosGetTimestampUs(); + SQueryTableDataCond* pQueryCond = taosArrayGet(pInfo->queryConds, readIdx); + int64_t st = taosGetTimestampUs(); void* p = tableListGetInfo(pTaskInfo->pTableInfoList, readIdx + pInfo->tableStartIndex); SReadHandle* pHandle = &pInfo->base.readHandle; - int32_t code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + int32_t code = + tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &pInfo->base.dataReader, GET_TASKID(pTaskInfo)); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } @@ -2533,18 +2538,11 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } // process this data block based on the probabilities - bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample); + bool processThisBlock = processBlockWithProbability(&pInfo->sample); if (!processThisBlock) { continue; } - blockDataCleanup(pBlock); - - int32_t rows = 0; - tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window); - blockDataEnsureCapacity(pBlock, rows); - pBlock->info.rows = rows; - if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; } else { @@ -2552,7 +2550,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } uint32_t status = 0; - loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + loadDataBlock(pOperator, &pInfo->base, pBlock, &status); // code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -2566,7 +2564,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); pOperator->resultInfo.totalRows += pBlock->info.rows; - pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; @@ -2646,6 +2644,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { param.readerIdx = i; param.pOperator = pOperator; param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); + blockDataEnsureCapacity(param.inputBlock, pOperator->resultInfo.capacity); + taosArrayPush(pInfo->sortSourceParams, ¶m); SQueryTableDataCond cond; @@ -2902,8 +2902,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, - getTableMergeScanExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableMergeScan, NULL, destroyTableMergeScanOperatorInfo, + optrDefaultBufFn, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; @@ -2913,3 +2913,353 @@ _error: taosMemoryFree(pOperator); return NULL; } + +// ==================================================================================================================== +// TableCountScanOperator +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator); +static void destoryTableCountScanOperator(void* param); +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid); +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName); +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName); +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName); +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes); +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum); +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum); +static const char* GROUP_TAG_DB_NAME = "db_name"; +static const char* GROUP_TAG_STABLE_NAME = "stable_name"; + +int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) { + if (scanCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, scanCols) { + if (nodeType(pNode) != QUERY_NODE_TARGET) { + return TSDB_CODE_QRY_SYS_ERROR; + } + STargetNode* targetNode = (STargetNode*)pNode; + if (nodeType(targetNode->pExpr) != QUERY_NODE_COLUMN) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SColumnNode* colNode = (SColumnNode*)(targetNode->pExpr); + if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) { + supp->dbNameSlotId = targetNode->slotId; + } else if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) { + supp->stbNameSlotId = targetNode->slotId; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tblCountScanGetCountSlotId(const SNodeList* pseudoCols, STableCountScanSupp* supp) { + if (pseudoCols != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, pseudoCols) { + if (nodeType(pNode) != QUERY_NODE_TARGET) { + return TSDB_CODE_QRY_SYS_ERROR; + } + STargetNode* targetNode = (STargetNode*)pNode; + if (nodeType(targetNode->pExpr) != QUERY_NODE_FUNCTION) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SFunctionNode* funcNode = (SFunctionNode*)(targetNode->pExpr); + if (funcNode->funcType == FUNCTION_TYPE_TABLE_COUNT) { + supp->tbCountSlotId = targetNode->slotId; + } + } + } + return TSDB_CODE_SUCCESS; +} + +int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCountScanSupp* supp) { + if (groupTags != NULL) { + SNode* pNode = NULL; + FOREACH(pNode, groupTags) { + if (nodeType(pNode) != QUERY_NODE_COLUMN) { + return TSDB_CODE_QRY_SYS_ERROR; + } + SColumnNode* colNode = (SColumnNode*)pNode; + if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) { + supp->groupByDbName = true; + } + if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) { + supp->groupByStbName = true; + } + } + } else { + strncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN); + strncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN); + } + return TSDB_CODE_SUCCESS; +} + +int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* scanCols, SNodeList* pseudoCols, + STableCountScanSupp* supp, SExecTaskInfo* taskInfo) { + int32_t code = 0; + code = tblCountScanGetInputs(groupTags, tableName, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo)); + return code; + } + supp->dbNameSlotId = -1; + supp->stbNameSlotId = -1; + supp->tbCountSlotId = -1; + + code = tblCountScanGetGroupTagsSlotId(scanCols, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo)); + return code; + } + code = tblCountScanGetCountSlotId(pseudoCols, supp); + if (code != TSDB_CODE_SUCCESS) { + qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo)); + return code; + } + return code; +} + +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pTblCountScanNode, + SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; + + SScanPhysiNode* pScanNode = &pTblCountScanNode->scan; + STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + + if (!pInfo || !pOperator) { + goto _error; + } + + pInfo->readHandle = *readHandle; + + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + initResultSizeInfo(&pOperator->resultInfo, 1); + pInfo->pRes = createDataBlockFromDescNode(pDescNode); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + getTableCountScanSupp(pTblCountScanNode->pGroupTags, &pTblCountScanNode->scan.tableName, + pTblCountScanNode->scan.pScanCols, pTblCountScanNode->scan.pScanPseudoCols, &pInfo->supp, + pTaskInfo); + + setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableCountScan, NULL, destoryTableCountScanOperator, + optrDefaultBufFn, NULL); + return pOperator; + +_error: + if (pInfo != NULL) { + destoryTableCountScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count, + SSDataBlock* pRes) { + if (pSupp->dbNameSlotId != -1) { + ASSERT(strlen(dbName)); + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId); + + char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(varDbName), dbName, TSDB_DB_NAME_LEN); + + varDataSetLen(varDbName, strlen(dbName)); + colDataAppend(colInfoData, 0, varDbName, false); + } + + if (pSupp->stbNameSlotId != -1) { + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId); + if (strlen(stbName) != 0) { + char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + strncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN); + varDataSetLen(varStbName, strlen(stbName)); + colDataAppend(colInfoData, 0, varStbName, false); + } else { + colDataAppendNULL(colInfoData, 0); + } + } + + if (pSupp->tbCountSlotId != -1) { + SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId); + colDataAppend(colInfoData, 0, (char*)&count, false); + } + pRes->info.rows = 1; +} + +static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo) { + STableCountScanSupp* pSupp = &pInfo->supp; + SSDataBlock* pRes = pInfo->pRes; + + size_t infodbTableNum; + getInfosDbMeta(NULL, &infodbTableNum); + size_t perfdbTableNum; + getPerfDbMeta(NULL, &perfdbTableNum); + + if (pSupp->groupByDbName) { + buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum); + return (pRes->info.rows > 0) ? pRes : NULL; + } else { + buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum); + return (pRes->info.rows > 0) ? pRes : NULL; + } +} + +static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes, + size_t infodbTableNum, size_t perfdbTableNum) { + if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) { + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else if (strlen(pSupp->dbNameFilter) == 0) { + fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes); + } + setOperatorCompleted(pOperator); +} + +static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum, + size_t perfdbTableNum) { + if (pInfo->currGrpIdx == 0) { + uint64_t groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes); + } else if (pInfo->currGrpIdx == 1) { + uint64_t groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB)); + pRes->info.id.groupId = groupId; + fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes); + } else { + setOperatorCompleted(pOperator); + } + pInfo->currGrpIdx++; +} + +static SSDataBlock* doTableCountScan(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableCountScanOperatorInfo* pInfo = pOperator->info; + STableCountScanSupp* pSupp = &pInfo->supp; + SSDataBlock* pRes = pInfo->pRes; + blockDataCleanup(pRes); + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + if (pInfo->readHandle.mnd != NULL) { + return buildSysDbTableCount(pOperator, pInfo); + } + + return buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes); +} + +static SSDataBlock* buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes) { + const char* db = NULL; + int32_t vgId = 0; + char dbName[TSDB_DB_NAME_LEN] = {0}; + + // get dbname + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + SName sn = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&sn, dbName); + + if (pSupp->groupByDbName) { + buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName); + } else { + buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName); + } + return pRes->info.rows > 0 ? pRes : NULL; +} + +static void buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) { + if (pSupp->groupByStbName) { + if (pInfo->stbUidList == NULL) { + pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t)); + if (vnodeGetStbIdList(pInfo->readHandle.vnode, 0, pInfo->stbUidList) < 0) { + qError("vgId:%d, failed to get stb id list error: %s", vgId, terrstr()); + } + } + if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) { + tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx); + buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid); + + pInfo->currGrpIdx++; + } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) { + buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName); + + pInfo->currGrpIdx++; + } else { + setOperatorCompleted(pOperator); + } + } else { + uint64_t groupId = calcGroupId(dbName, strlen(dbName)); + pRes->info.id.groupId = groupId; + int64_t dbTableCount = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes); + setOperatorCompleted(pOperator); + } +} + +static void buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo, + STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) { + if (strlen(pSupp->dbNameFilter) != 0) { + if (strlen(pSupp->stbNameFilter) != 0) { + tb_uid_t uid = metaGetTableEntryUidByName(pInfo->readHandle.meta, pSupp->stbNameFilter); + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, uid, &stats); + int64_t ctbNum = stats.ctbNum; + fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, ctbNum, pRes); + } else { + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); + } + } else { + int64_t tbNumVnode = metaGetTbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes); + } + setOperatorCompleted(pOperator); +} + +static void buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName) { + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, ""); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + int64_t ntbNum = metaGetNtbNum(pInfo->readHandle.meta); + fillTableCountScanDataBlock(pSupp, dbName, "", ntbNum, pRes); +} + +static void buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp, + SSDataBlock* pRes, char* dbName, tb_uid_t stbUid) { + char stbName[TSDB_TABLE_NAME_LEN] = {0}; + metaGetTableSzNameByUid(pInfo->readHandle.meta, stbUid, stbName); + + char fullStbName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, stbName); + uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName)); + pRes->info.id.groupId = groupId; + + SMetaStbStats stats = {0}; + metaGetStbStats(pInfo->readHandle.meta, stbUid, &stats); + int64_t ctbNum = stats.ctbNum; + + fillTableCountScanDataBlock(pSupp, dbName, stbName, ctbNum, pRes); +} + +static void destoryTableCountScanOperator(void* param) { + STableCountScanOperatorInfo* pTableCountScanInfo = param; + blockDataDestroy(pTableCountScanInfo->pRes); + + taosArrayDestroy(pTableCountScanInfo->stbUidList); + taosMemoryFreeClear(param); +} diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index ec754f31b0ab3af59a91cb863de6ede52e0ccb4a..7ac007b7cb914f5b4fe3faa8110d3acd0b72a852 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -75,7 +75,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* // TODO dynamic set the available sort buffer pOperator->fpSet = - createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, getExplainExecInfo); + createOperatorFpSet(doOpenSortOperator, doSort, NULL, destroySortOperatorInfo, optrDefaultBufFn, getExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -105,6 +105,7 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { } } + pBlock->info.dataLoad = 1; pBlock->info.rows += 1; } @@ -521,8 +522,8 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); setOperatorInfo(pOperator, "GroupSortOperator", QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, - getGroupSortExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doGroupSort, NULL, destroyGroupSortOperatorInfo, + optrDefaultBufFn, getGroupSortExplainExecInfo); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -559,7 +560,7 @@ typedef struct SMultiwayMergeOperatorInfo { STupleHandle* prefetchedTuple; } SMultiwayMergeOperatorInfo; -int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { +int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { SMultiwayMergeOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -577,9 +578,15 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SOperatorInfo* pDownstream = pOperator->pDownstream[i]; + if (pDownstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { + pDownstream->fpSet._openFn(pDownstream); + } + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = pOperator->pDownstream[i]; + ps->param = pDownstream; ps->onlyRef = true; + tsortAddSource(pInfo->pSortHandle, ps); } @@ -692,6 +699,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData pInfo->limitInfo.numOfOutputRows += p->info.rows; pDataBlock->info.rows = p->info.rows; pDataBlock->info.id.groupId = pInfo->groupId; + pDataBlock->info.dataLoad = 1; } qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, @@ -714,7 +722,6 @@ SSDataBlock* doMultiwayMerge(SOperatorInfo* pOperator) { } qDebug("start to merge final sorted rows, %s", GET_TASKID(pTaskInfo)); - SSDataBlock* pBlock = getMultiwaySortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->matchInfo.pList, pOperator); if (pBlock != NULL) { pOperator->resultInfo.totalRows += pBlock->info.rows; @@ -781,7 +788,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0); SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc); - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 1024); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); pInfo->groupSort = pMergePhyNode->groupSort; @@ -792,8 +799,8 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(doOpenMultiwayMergeOperator, doMultiwayMerge, NULL, - destroyMultiwayMergeOperatorInfo, getMultiwayMergeExplainExecInfo); + pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, + destroyMultiwayMergeOperatorInfo, optrDefaultBufFn, getMultiwayMergeExplainExecInfo); code = appendDownstream(pOperator, downStreams, numStreams); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 3d35326749ed2ccd09d6473abe6424fe927d3dd0..a88f673e0b33135f41c1fa5e868d61c66942ee9f 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -1335,7 +1335,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -1437,7 +1437,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, optrDefaultBufFn, NULL); return pOperator; _error: @@ -1874,78 +1874,80 @@ static void destroyBlockDistScanOperatorInfo(void* param) { } static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { - memset(pCond, 0, sizeof(SQueryTableDataCond)); - - pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = 1; - pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return terrno; - } + memset(pCond, 0, sizeof(SQueryTableDataCond)); + + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + pCond->pSlotList = taosMemoryMalloc(sizeof(int32_t)); + if (pCond->colList == NULL || pCond->pSlotList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } - pCond->colList->colId = 1; - pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; - pCond->colList->bytes = sizeof(TSKEY); + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = uid; - pCond->type = TIMEWINDOW_RANGE_CONTAINED; - pCond->startVersion = -1; - pCond->endVersion = -1; + pCond->pSlotList[0] = 0; - return TSDB_CODE_SUCCESS; + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; } SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo) { - SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } + SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } - { - SQueryTableDataCond cond = {0}; + pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pResBlock, 1); - int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + { + SQueryTableDataCond cond = {0}; + int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); - cleanupQueryTableDataCond(&cond); - if (code != 0) { - goto _error; - } + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; } + } - pInfo->readHandle = *readHandle; - pInfo->uid = pBlockScanNode->suid; - - pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pResBlock, 1); + pInfo->readHandle = *readHandle; + pInfo->uid = pBlockScanNode->suid; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, - OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); - return pOperator; + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, optrDefaultBufFn, NULL); + return pOperator; - _error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; +_error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; } \ No newline at end of file diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index a66a5e70892f91adb60648fa2eb5d0f8b8961bb9..d30c1fbfa143f2cbe4bbdc4cc1e3cd981f0ba9de 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -45,8 +45,18 @@ static void setNullRow(SSDataBlock* pBlock, SFillInfo* pFillInfo, int32_t rowInd if (pCol->notFillCol) { bool filled = fillIfWindowPseudoColumn(pFillInfo, pCol, pDstColInfo, rowIndex); if (!filled) { - SArray* p = FILL_IS_ASC_FILL(pFillInfo) ? pFillInfo->prev.pRowVal : pFillInfo->next.pRowVal; - SGroupKeys* pKey = taosArrayGet(p, i); + SRowVal* p = NULL; + if (FILL_IS_ASC_FILL(pFillInfo)) { + if (pFillInfo->prev.key != 0) { + p = &pFillInfo->prev; // prev has been set value + } else { // otherwise, use the value in the next row + p = &pFillInfo->next; + } + } else { + p = &pFillInfo->next; + } + + SGroupKeys* pKey = taosArrayGet(p->pRowVal, i); doSetVal(pDstColInfo, rowIndex, pKey); } } else { @@ -246,7 +256,10 @@ static void initBeforeAfterDataBuf(SFillInfo* pFillInfo) { static void saveColData(SArray* rowBuf, int32_t columnIndex, const char* src, bool isNull); -static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray* pRow) { +static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVal* pRowVal) { + SColumnInfoData* pTsCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, pFillInfo->srcTsSlotId); + pRowVal->key = ((int64_t*)pTsCol->pData)[rowIndex]; + for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) { int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType; if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) { @@ -257,7 +270,7 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SArray bool isNull = colDataIsNull_s(pSrcCol, rowIndex); char* p = colDataGetData(pSrcCol, rowIndex); - saveColData(pRow, i, p, isNull); + saveColData(pRowVal->pRowVal, i, p, isNull); } else { ASSERT(0); } @@ -281,7 +294,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t // set the next value for interpolation if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) { - copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next); } if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) && @@ -303,7 +316,7 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) { int32_t nextRowIndex = pFillInfo->index + 1; - copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, pFillInfo->next.pRowVal); + copyCurrentRowIntoBuf(pFillInfo, nextRowIndex, &pFillInfo->next); } // copy rows to dst buffer @@ -319,6 +332,9 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t if (!colDataIsNull_s(pSrc, pFillInfo->index)) { colDataAppend(pDst, index, src, false); saveColData(pFillInfo->prev.pRowVal, i, src, false); + if (pFillInfo->srcTsSlotId == dstSlotId) { + pFillInfo->prev.key = *(int64_t*)src; + } } else { // the value is null if (pDst->info.type == TSDB_DATA_TYPE_TIMESTAMP) { colDataAppend(pDst, index, (const char*)&pFillInfo->currentKey, false); diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 90f5dde7c3ad0b979edb876f7cc613461c330c9d..a82dc13318f2b5c68cb2ed3895c78dd8f5df2696 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -49,10 +49,10 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock if (!colDataIsNull_s(pColInfoData, rowIndex)) { pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); - if (!IS_VAR_DATA_TYPE(pkey->type)) { - memcpy(pkey->pData, val, pkey->bytes); - } else { + if (IS_VAR_DATA_TYPE(pkey->type)) { memcpy(pkey->pData, val, varDataLen(val)); + } else { + memcpy(pkey->pData, val, pkey->bytes); } } else { pkey->isNull = true; @@ -91,18 +91,37 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); + + if (!IS_MATHABLE_TYPE(pColInfoData->info.type)) { + continue; + } + // null value is represented by using key = INT64_MIN for now. // TODO: optimize to ignore null values for linear interpolation. if (!pLinearInfo->isStartSet) { if (!colDataIsNull_s(pColInfoData, rowIndex)) { + pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + char* p = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); + memcpy(pLinearInfo->start.val, p, varDataTLen(p)); + } else { + memcpy(pLinearInfo->start.val, p, pLinearInfo->bytes); + } } pLinearInfo->isStartSet = true; } else if (!pLinearInfo->isEndSet) { if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + + char* p = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); + memcpy(pLinearInfo->end.val, p, varDataTLen(p)); + } else { + memcpy(pLinearInfo->end.val, p, pLinearInfo->bytes); + } } pLinearInfo->isEndSet = true; } else { @@ -111,7 +130,15 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo if (!colDataIsNull_s(pColInfoData, rowIndex)) { pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); - memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes); + + char* p = colDataGetData(pColInfoData, rowIndex); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + ASSERT(varDataTLen(p) <= pColInfoData->info.bytes); + memcpy(pLinearInfo->end.val, p, varDataTLen(p)); + } else { + memcpy(pLinearInfo->end.val, p, pLinearInfo->bytes); + } + } else { pLinearInfo->end.key = INT64_MIN; } @@ -544,7 +571,7 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5cf0d3111732c668c367b073c9a0cbff40e35644..ca64fd135eb05e4df97d9bd93e88df8e403dc349 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -23,6 +23,7 @@ #include "ttime.h" #define IS_FINAL_OP(op) ((op)->isFinal) +#define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); typedef struct SSessionAggOperatorInfo { SOptrBasicInfo binfo; @@ -56,6 +57,7 @@ typedef enum SResultTsInterpType { typedef struct SPullWindowInfo { STimeWindow window; uint64_t groupId; + STimeWindow calWin; } SPullWindowInfo; typedef struct SOpenWindowInfo { @@ -648,7 +650,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); @@ -793,17 +795,18 @@ int32_t comparePullWinKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; SPullWindowInfo* pos = taosArrayGet(res, index); SPullWindowInfo* pData = (SPullWindowInfo*)pKey; - if (pData->window.skey == pos->window.skey) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (pData->window.skey > pos->window.skey) { + if (pData->groupId > pos->groupId) { return 1; + } else if (pData->groupId < pos->groupId) { + return -1; + } + + if (pData->window.skey > pos->window.ekey) { + return 1; + } else if (pData->window.ekey < pos->window.skey) { + return -1; } - return -1; + return 0; } static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { @@ -812,10 +815,16 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { if (index == -1) { index = 0; } else { - if (comparePullWinKey(pPullInfo, pPullWins, index) > 0) { - index++; - } else { + int32_t code = comparePullWinKey(pPullInfo, pPullWins, index); + if (code == 0) { + SPullWindowInfo* pos = taosArrayGet(pPullWins ,index); + pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey); + pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey); + pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey); + pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey); return TSDB_CODE_SUCCESS; + } else if (code > 0 ){ + index++; } } if (taosArrayInsert(pPullWins, index, pPullInfo) == NULL) { @@ -863,19 +872,20 @@ static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { int32_t compareWinRes(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SResKeyPos* pData = (SResKeyPos*)pKey; - if (*(int64_t*)pData->key == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (*(int64_t*)pData->key > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SResKeyPos* pRKey = (SResKeyPos*)pKey; + if (pRKey->groupId > pDataPos->groupId) { return 1; + } else if (pRKey->groupId < pDataPos->groupId) { + return -1; } - return -1; + + if (*(int64_t*)pRKey->key > pDataPos->ts) { + return 1; + } else if (*(int64_t*)pRKey->key < pDataPos->ts){ + return -1; + } + return 0; } static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { @@ -927,7 +937,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = @@ -943,7 +953,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -967,7 +977,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -1027,9 +1037,10 @@ SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SRe int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) { TSKEY* tsCols = NULL; - if (pBlock->pDataBlock != NULL) { + if (pBlock->pDataBlock != NULL && pBlock->info.dataLoad == 1) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; + ASSERT(tsCols[0] != 0); // no data in primary ts if (tsCols[0] == 0 && tsCols[pBlock->info.rows - 1] == 0) { @@ -1049,14 +1060,14 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SIntervalAggOperatorInfo* pInfo = pOperator->info; SExprSupp* pSup = &pOperator->exprSupp; int32_t scanFlag = MAIN_SCAN; - - int64_t st = taosGetTimestampUs(); - SOperatorInfo* downstream = pOperator->pDownstream[0]; + int64_t st = taosGetTimestampUs(); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -1073,8 +1084,6 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, pInfo->inputOrder, scanFlag, true); - blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); } @@ -1150,7 +1159,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); @@ -1175,7 +1184,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -1268,15 +1277,11 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) { } SSDataBlock* pBlock = pInfo->binfo.pRes; - - ASSERT(pInfo->execModel == OPTR_EXEC_MODEL_BATCH); - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } - blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); @@ -1404,19 +1409,21 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { int32_t compareWinKey(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGet(res, index); - SWinKey* pData = (SWinKey*)pKey; - if (pData->ts == pos->ts) { - if (pData->groupId > pos->groupId) { - return 1; - } else if (pData->groupId < pos->groupId) { - return -1; - } - return 0; - } else if (pData->ts > pos->ts) { + SWinKey* pDataPos = taosArrayGet(res, index); + SWinKey* pWKey = (SWinKey*)pKey; + + if (pWKey->groupId > pDataPos->groupId) { return 1; + } else if (pWKey->groupId < pDataPos->groupId) { + return -1; + } + + if (pWKey->ts > pDataPos->ts) { + return 1; + } else if (pWKey->ts < pDataPos->ts) { + return -1; } - return -1; + return 0; } static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, @@ -1649,23 +1656,34 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) { // the primary timestamp column bool needed = false; - pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); - pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); - { // ts column - SColumn c = {0}; - c.colId = 1; - c.slotId = pInfo->primaryTsIndex; - c.type = TSDB_DATA_TYPE_TIMESTAMP; - c.bytes = sizeof(int64_t); - taosArrayPush(pInfo->pInterpCols, &c); + for(int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = pCtx[i].pExpr; + if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) { + needed = true; + break; + } + } + + if (needed) { + pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = true; // to denote no value is assigned yet - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevValues, &key); + { // ts column + SColumn c = {0}; + c.colId = 1; + c.slotId = pInfo->primaryTsIndex; + c.type = TSDB_DATA_TYPE_TIMESTAMP; + c.bytes = sizeof(int64_t); + taosArrayPush(pInfo->pInterpCols, &c); + + SGroupKeys key; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = true; // to denote no value is assigned yet + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(pInfo->pPrevValues, &key); + } } for (int32_t i = 0; i < numOfCols; ++i) { @@ -1676,7 +1694,6 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt SColumn c = *pParam->pCol; taosArrayPush(pInfo->pInterpCols, &c); - needed = true; SGroupKeys key = {0}; key.bytes = c.bytes; @@ -1708,7 +1725,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { for (int32_t i = 0; i < numOfExpr; i++) { - pCtx[i].isStream = true; +// pCtx[i].isStream = true; } } @@ -1727,7 +1744,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->primaryTsIndex = ((SColumnNode*)pPhyNode->window.pTspk)->slotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 512); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); @@ -1755,7 +1773,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo->inputOrder = (pPhyNode->window.inputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->resultTsOrder = (pPhyNode->window.outputTsOrder == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pInfo->interval = interval; - pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = as; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; @@ -1773,11 +1790,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh goto _error; } - if (isStream) { - ASSERT(num > 0); - initStreamFunciton(pSup->pCtx, pSup->numOfExprs); - } - initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, num, pInfo); if (pInfo->timeWindowInterpo) { @@ -1792,7 +1804,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, NULL); + createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -1854,7 +1866,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } // pInfo->numOfRows data belong to the current session window @@ -1873,7 +1885,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); @@ -2010,7 +2022,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, NULL); + createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2083,7 +2095,7 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2163,7 +2175,7 @@ static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, S int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } num++; @@ -2251,8 +2263,8 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); - colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + colDataAppend(pCalStartTs, pBlock->info.rows, (const char*)&pWin->calWin.skey, false); + colDataAppend(pCalEndTs, pBlock->info.rows, (const char*)&pWin->calWin.ekey, false); pBlock->info.rows++; } if ((*pIndex) == size) { @@ -2262,27 +2274,33 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB blockDataUpdateTsWindow(pBlock, 0); } -void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { +void processPullOver(SSDataBlock* pBlock, SHashObj* pMap, SInterval* pInterval) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* tsEndData = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { - SWinKey winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; - void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey)); - if (chIds) { - SArray* chArray = *(SArray**)chIds; - int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); - if (index != -1) { - qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); - taosArrayRemove(chArray, index); - if (taosArrayGetSize(chArray) == 0) { - // pull data is over - taosArrayDestroy(chArray); - taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + TSKEY winTs = tsData[i]; + while (winTs < tsEndData[i]) { + SWinKey winRes = {.ts = winTs, .groupId = groupIdData[i]}; + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinKey)); + if (chIds) { + SArray* chArray = *(SArray**)chIds; + int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + if (index != -1) { + qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosArrayDestroy(chArray); + taosHashRemove(pMap, &winRes, sizeof(SWinKey)); + } } } + winTs = taosTimeAdd(winTs, pInterval->sliding, pInterval->slidingUnit, pInterval->precision); } } } @@ -2295,12 +2313,13 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId}; + SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - savePullWindow(&pull, pInfo->pPullWins); - int32_t size1 = taosArrayGetSize(pInfo->pChildren); - addPullWindow(pInfo->pPullDataMap, winKey, size1); - qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1); + if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { + int32_t size1 = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, winKey, size1); + qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1); + } } } } @@ -2370,12 +2389,13 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = groupId}; + SPullWindowInfo pull = {.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey}; // add pull data request - savePullWindow(&pull, pInfo->pPullWins); - int32_t size = taosArrayGetSize(pInfo->pChildren); - addPullWindow(pInfo->pPullDataMap, &winRes, size); - qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); + if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) { + int32_t size = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, &winRes, size); + qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); + } } else { int32_t index = -1; SArray* chArray = NULL; @@ -2402,7 +2422,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } if (IS_FINAL_OP(pInfo)) { @@ -2556,7 +2576,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { - processPullOver(pBlock, pInfo->pPullDataMap); + processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); continue; } @@ -2573,7 +2593,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { for (int32_t i = 0; i < chIndex + 1 - size; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; @@ -2634,6 +2654,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return NULL; } +int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) { + if (pIntervalPhyNode->window.deleteMark <= 0) { + return DEAULT_DELETE_MARK; + } + int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark,pIntervalPhyNode->window.watermark); + deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval); + return deleteMark; +} + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -2655,9 +2684,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, .minTs = INT64_MAX, - // for test 315360000000 - .deleteMark = 1000LL * 60LL * 60LL * 24LL * 365LL * 10LL, - // .deleteMark = INT64_MAX, + .deleteMark = getDeleteMark(pIntervalPhyNode), .deleteMarkSaved = 0, .calTriggerSaved = 0, }; @@ -2744,7 +2771,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); } @@ -2999,7 +3026,7 @@ static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindo SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t code = initSessionOutputBuf(pCurWin, pResult, pSup->pCtx, numOutput, pSup->rowEntryInfoOffset); if (code != TSDB_CODE_SUCCESS || (*pResult) == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } updateTimeWindowInfo(pTimeWindowData, &pCurWin->sessionWin.win, false); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, pTimeWindowData, startIndex, winRows, rows, numOutput); @@ -3108,13 +3135,13 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData pAggSup->pResultRows, pStUpdated, pStDeleted); // coverity scan error if (!winInfo.pOutputBuf) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted); saveSessionOutputBuf(pAggSup, &winInfo); @@ -3122,7 +3149,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -3446,7 +3473,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { - T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } taosArrayPush(pInfo->pChildren, &pChildOp); } @@ -3556,7 +3583,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); if (downstream) { initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); @@ -3700,8 +3727,8 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream if (pPhyNode->type != QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); blockDataEnsureCapacity(pInfo->pUpdateRes, 128); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, - destroyStreamSessionAggOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionSemiAgg, NULL, + destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL); } setOperatorInfo(pOperator, name, pPhyNode->type, false, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3889,14 +3916,14 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } saveSessionOutputBuf(pAggSup, &curWin.winInfo); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -4061,7 +4088,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL); initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4309,12 +4336,11 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, iaInfo->win = pTaskInfo->window; iaInfo->inputOrder = TSDB_ORDER_ASC; iaInfo->interval = interval; - iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - initResultSizeInfo(&pOperator->resultInfo, 4096); + initResultSizeInfo(&pOperator->resultInfo, 512); int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); @@ -4339,7 +4365,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, false, OP_NOT_OPENED, miaInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4448,7 +4474,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } TSKEY ekey = ascScan ? win.ekey : win.skey; @@ -4465,7 +4491,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } // window start key interpolation @@ -4494,7 +4520,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } ekey = ascScan ? nextWin.ekey : nextWin.skey; @@ -4615,7 +4641,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pIntervalInfo->win = pTaskInfo->window; pIntervalInfo->inputOrder = TSDB_ORDER_ASC; pIntervalInfo->interval = interval; - pIntervalInfo->execModel = pTaskInfo->execModel; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -4645,7 +4670,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false, OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo); pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, NULL); + createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -4803,7 +4828,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, .minTs = INT64_MAX, - .deleteMark = INT64_MAX, + .deleteMark = getDeleteMark(pIntervalPhyNode), }; ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); @@ -4861,8 +4886,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, NULL); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamIntervalAgg, NULL, + destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); code = appendDownstream(pOperator, &downstream, 1); diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 42046925142f2d4957493679f3e0c62bcb31a943..d97f81c9946db6928f87a83d0a12896c85b6054a 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -17,6 +17,7 @@ #include "taoserror.h" #include "tdef.h" #include "tpagedbuf.h" +#include "tlog.h" #define LHASH_CAP_RATIO 0.85 diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index d5a2f7717547a3abbec08f3c36d8941f22b60129..30911887bbb42dd9e116f8c3cea2e587bc456144 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -34,14 +34,12 @@ struct SSortHandle { int32_t pageSize; int32_t numOfPages; SDiskbasedBuf* pBuf; - - SArray* pSortInfo; - SArray* pOrderedSource; - - int32_t loops; - uint64_t sortElapsed; - int64_t startTs; - uint64_t totalElapsed; + SArray* pSortInfo; + SArray* pOrderedSource; + int32_t loops; + uint64_t sortElapsed; + int64_t startTs; + uint64_t totalElapsed; int32_t sourceId; SSDataBlock* pDataBlock; @@ -99,9 +97,9 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page } static int32_t sortComparCleanup(SMsortComparParam* cmpParam) { + // NOTICE: pSource may be, if it is SORT_MULTISOURCE_MERGE for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = - cmpParam->pSources[i]; // NOTICE: pSource may be SGenericSource *, if it is SORT_MULTISOURCE_MERGE + SSortSource* pSource = cmpParam->pSources[i]; blockDataDestroy(pSource->src.pBlock); taosMemoryFreeClear(pSource); } @@ -158,7 +156,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource)); if (pSource == NULL) { taosArrayDestroy(pPageIdList); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pSource->src.pBlock = pBlock; @@ -231,15 +229,15 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList); } -static void setCurrentSourceIsDone(SSortSource* pSource, SSortHandle* pHandle) { +static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) { pSource->src.rowIndex = -1; ++pHandle->numOfCompletedSources; } -static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, +static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { - cmpParam->pSources = taosArrayGet(pSources, startIndex); - cmpParam->numOfSources = (endIndex - startIndex + 1); + pParam->pSources = taosArrayGet(pSources, startIndex); + pParam->numOfSources = (endIndex - startIndex + 1); int32_t code = 0; @@ -247,7 +245,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { code = TSDB_CODE_NO_AVAIL_DISK; - qError("Sort compare init failed since %s", terrstr(code)); + qError("Sort compare init failed since %s, %s", terrstr(code), pHandle->idStr); return code; } @@ -260,12 +258,12 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } if (pHandle->type == SORT_SINGLESOURCE_SORT) { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; // set current source is done if (taosArrayGetSize(pSource->pageIdList) == 0) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); continue; } @@ -280,15 +278,21 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int releaseBufPage(pHandle->pBuf, pPage); } } else { - for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SSortSource* pSource = cmpParam->pSources[i]; + qDebug("start init for the multiway merge sort, %s", pHandle->idStr); + int64_t st = taosGetTimestampUs(); + + for (int32_t i = 0; i < pParam->numOfSources; ++i) { + SSortSource* pSource = pParam->pSources[i]; pSource->src.pBlock = pHandle->fetchfp(pSource->param); // set current source is done if (pSource->src.pBlock == NULL) { - setCurrentSourceIsDone(pSource, pHandle); + setCurrentSourceDone(pSource, pHandle); } } + + int64_t et = taosGetTimestampUs(); + qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 1c4216334945c0b682e313a975e558390fbd7049..2efd06f4401b94968e493aa79775544e01bf11b7 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -166,6 +166,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { pInfo->current += 1; + pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); return pBlock; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fe010786ebdcb12d9df1350fbe565c6fb1494e3f..07e480ee1d5b6c5c5cad3c8a56472ae5f262e94e 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1924,7 +1924,8 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int } static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (2 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (2 != numOfParams && 3 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1935,9 +1936,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); if (ret == TIME_UNIT_TOO_SMALL) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, @@ -1948,11 +1947,30 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } + if (3 == numOfParams) { + uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type; + if (!IS_INTEGER_TYPE(para3Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2); + if (pValue->datum.i != 0 && pValue->datum.i != 1) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + // add database precision as param + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } + // add client timezone as param + code = addTimezoneParam(pFunc->pParameterList); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; @@ -2080,6 +2098,11 @@ static int32_t translateTagsPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in return TSDB_CODE_SUCCESS; } +static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + return TSDB_CODE_SUCCESS; +} + // clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { @@ -3241,6 +3264,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "_table_count", + .type = FUNCTION_TYPE_TABLE_COUNT, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, + .translateFunc = translateTableCountPseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 20de7f73bf6baa993885469f85838c6f46386482..111da6d6ba563b6b9b3166de7940f919b1581326 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -423,6 +423,8 @@ typedef struct SGroupKeyInfo { (_p).val = (_v); \ } while (0) +static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst); + bool functionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; @@ -457,11 +459,12 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts > pSBuf->ts)) { - memcpy(pDBuf->buf, pSBuf->buf, bytes); - pDBuf->ts = pSBuf->ts; - pDResInfo->numOfRes = 1; + if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, true)) { + pDBuf->hasResult = true; } + + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; } @@ -501,7 +504,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; ASSERT(numOfElem >= 0); } else { @@ -1274,17 +1277,8 @@ int32_t stddevCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SStddevRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type; - if (IS_SIGNED_NUMERIC_TYPE(type)) { - pDBuf->isum += pSBuf->isum; - pDBuf->quadraticISum += pSBuf->quadraticISum; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - pDBuf->usum += pSBuf->usum; - pDBuf->quadraticUSum += pSBuf->quadraticUSum; - } else { - pDBuf->dsum += pSBuf->dsum; - pDBuf->quadraticDSum += pSBuf->quadraticDSum; - } - pDBuf->count += pSBuf->count; + stddevTransferInfo(pSBuf, pDBuf); + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; @@ -2289,16 +2283,15 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, - int32_t rowIndex) { +static int32_t firstLastTransferInfoImpl(SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) { if (pOutput->hasResult) { if (isFirst) { if (pInput->ts > pOutput->ts) { - return; + return TSDB_CODE_FAILED; } } else { if (pInput->ts < pOutput->ts) { - return; + return TSDB_CODE_FAILED; } } } @@ -2308,9 +2301,15 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S pOutput->bytes = pInput->bytes; memcpy(pOutput->buf, pInput->buf, pOutput->bytes); - firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + return TSDB_CODE_SUCCESS; +} - pOutput->hasResult = true; +static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst, + int32_t rowIndex) { + if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pInput, pOutput, isFirst)) { + firstlastSaveTupleData(pCtx->pSrcBlock, rowIndex, pCtx, pOutput); + pOutput->hasResult = true; + } } static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) { @@ -2378,7 +2377,6 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return 1; } -// todo rewrite: int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); SFirstLastRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo); @@ -2387,11 +2385,12 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx); SFirstLastRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo); - if (pSResInfo->numOfRes != 0 && (pDResInfo->numOfRes == 0 || pDBuf->ts < pSBuf->ts)) { - memcpy(pDBuf->buf, pSBuf->buf, bytes); - pDBuf->ts = pSBuf->ts; - pDResInfo->numOfRes = 1; + if (TSDB_CODE_SUCCESS == firstLastTransferInfoImpl(pSBuf, pDBuf, false)) { + pDBuf->hasResult = true; } + + pDResInfo->numOfRes = TMAX(pDResInfo->numOfRes, pSResInfo->numOfRes); + pDResInfo->isNullRes &= pSResInfo->isNullRes; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/detail/tavgfunction.c b/source/libs/function/src/detail/tavgfunction.c index 40698322a0a10ebab6704c3451f024ba353da140..60bf30d8edb524b1a2c17955bca830f1b4f34213 100644 --- a/source/libs/function/src/detail/tavgfunction.c +++ b/source/libs/function/src/detail/tavgfunction.c @@ -514,7 +514,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { numOfElem = pInput->numOfRows; pAvgRes->count += pInput->numOfRows; - bool simdAvailable = tsAVXEnable && tsSIMDEnable && (numOfRows > THRESHOLD_SIZE); + bool simdAvailable = tsAVXEnable && tsSIMDBuiltins && (numOfRows > THRESHOLD_SIZE); switch(type) { case TSDB_DATA_TYPE_UTINYINT: @@ -586,7 +586,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { if (type == TSDB_DATA_TYPE_BIGINT) { pAvgRes->sum.isum += plist[i]; } else { - pAvgRes->sum.isum += (uint64_t)plist[i]; + pAvgRes->sum.usum += (uint64_t)plist[i]; } } } diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index b2cb36cba07d5414994ecd98683b63188b835a60..cb5cea3cc851692a2a348ff2319f00d96b360119 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -369,7 +369,7 @@ static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { // AVX2 version to speedup the loop - if (tsAVX2Enable && tsSIMDEnable) { + if (tsAVX2Enable && tsSIMDBuiltins) { pBuf->v = i8VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); } else { if (!pBuf->assign) { @@ -403,7 +403,7 @@ static void handleInt8Col(const void* data, int32_t start, int32_t numOfRows, SM static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { // AVX2 version to speedup the loop - if (tsAVX2Enable && tsSIMDEnable) { + if (tsAVX2Enable && tsSIMDBuiltins) { pBuf->v = i16VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); } else { if (!pBuf->assign) { @@ -437,7 +437,7 @@ static void handleInt16Col(const void* data, int32_t start, int32_t numOfRows, S static void handleInt32Col(const void* data, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc, bool signVal) { // AVX2 version to speedup the loop - if (tsAVX2Enable && tsSIMDEnable) { + if (tsAVX2Enable && tsSIMDBuiltins) { pBuf->v = i32VectorCmpAVX2(data, numOfRows, isMinFunc, signVal); } else { if (!pBuf->assign) { @@ -500,7 +500,7 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo float* val = (float*)&pBuf->v; // AVX version to speedup the loop - if (tsAVXEnable && tsSIMDEnable) { + if (tsAVXEnable && tsSIMDBuiltins) { *val = floatVectorCmpAVX(pData, numOfRows, isMinFunc); } else { if (!pBuf->assign) { @@ -530,7 +530,7 @@ static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfR double* val = (double*)&pBuf->v; // AVX version to speedup the loop - if (tsAVXEnable && tsSIMDEnable) { + if (tsAVXEnable && tsSIMDBuiltins) { *val = (double)doubleVectorCmpAVX(pData, numOfRows, isMinFunc); } else { if (!pBuf->assign) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index e5727f14723f78f4dee1a93a0de0259bddc382f5..157ee08f1522118e0ab12daa5426c1f44c00adda 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -22,6 +22,7 @@ #include "tpagedbuf.h" #include "tpercentile.h" #include "ttypes.h" +#include "tlog.h" #define DEFAULT_NUM_OF_SLOT 1024 @@ -363,15 +364,18 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); // keep the pointer in memory + setBufPageDirty(pSlot->info.data, true); releaseBufPage(pBucket->pBuffer, pSlot->info.data); pSlot->info.data = NULL; } - SArray *pPageIdList = (SArray *)taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); - if (pPageIdList == NULL) { - SArray *pList = taosArrayInit(4, sizeof(int32_t)); - taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pList, POINTER_BYTES); - pPageIdList = pList; + SArray *pPageIdList; + void *p = taosHashGet(pBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p == NULL) { + pPageIdList = taosArrayInit(4, sizeof(int32_t)); + taosHashPut(pBucket->groupPagesMap, &groupId, sizeof(groupId), &pPageIdList, POINTER_BYTES); + } else { + pPageIdList = *(SArray **)p; } pSlot->info.data = getNewBufPage(pBucket->pBuffer, &pageId); @@ -494,15 +498,16 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) resetSlotInfo(pMemBucket); int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SArray* list = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + SArray* list = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); ASSERT(list != NULL && list->size > 0); for (int32_t f = 0; f < list->size; ++f) { - SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); - SFilePage *pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo)); + int32_t *pageId = taosArrayGet(list, f); + SFilePage *pg = getBufPage(pMemBucket->pBuffer, *pageId); tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); - releaseBufPageInfo(pMemBucket->pBuffer, pgInfo); + setBufPageDirty(pg, true); + releaseBufPage(pMemBucket->pBuffer, pg); } return getPercentileImpl(pMemBucket, count - num, fraction); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index c78ec5b9991afb7bc9de0f286ab685c99d6d9595..0b309bc8f56abd3870a07eebdb94e61f357ee23c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -88,11 +88,13 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { } #ifdef WINDOWS if (strlen(path) == 0) { - strcat(path, "udfd.exe"); - } else { - strcat(path, "\\udfd.exe"); + strcat(path, "C:\\TDengine"); } + strcat(path, "\\udfd.exe"); #else + if (strlen(path) == 0) { + strcat(path, "/usr/bin"); + } strcat(path, "/udfd"); #endif char *argsUdfd[] = {path, "-c", configDir, NULL}; @@ -835,9 +837,34 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { } int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { - output->info.rows = input->numOfRows; + int32_t numOfRows = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + numOfRows = (input[i].numOfRows > numOfRows) ? input[i].numOfRows : numOfRows; + } + output->info.rows = numOfRows; output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { + if ((input+i)->numOfRows < numOfRows) { + SColumnInfoData* pColInfoData = (input+i)->columnData; + int32_t startRow = (input+i)->numOfRows; + int32_t expandRows = numOfRows - startRow; + colInfoDataEnsureCapacity(pColInfoData, numOfRows, false); + bool isNull = colDataIsNull_s(pColInfoData, (input+i)->numOfRows - 1); + if (isNull) { + colDataAppendNNULL(pColInfoData, startRow, expandRows); + } else { + char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1); + int32_t bytes = pColInfoData->info.bytes; + char* data = taosMemoryMalloc(bytes); + memcpy(data, src, bytes); + for (int j = 0; j < expandRows; ++j) { + colDataAppend(pColInfoData, startRow+j, data, false); + } + //colDataAppendNItems(pColInfoData, startRow, data, expandRows); + taosMemoryFree(data); + } + } + taosArrayPush(output->pDataBlock, (input + i)->columnData); if (IS_VAR_DATA_TYPE((input + i)->columnData->info.type)) { diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index d87df2d0f335a3901960491ce232eecfaaa5587e..40c75ce6baacbaad258ad10874af16f9466ad2d4 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -28,39 +28,46 @@ #include "tmsg.h" #include "trpc.h" #include "tmisce.h" -// clang-foramt on +// clang-format on typedef struct SUdfdContext { - uv_loop_t * loop; + uv_loop_t *loop; uv_pipe_t ctrlPipe; uv_signal_t intrSignal; char listenPipeName[PATH_MAX + UDF_LISTEN_PIPE_NAME_LEN + 2]; uv_pipe_t listeningPipe; - void * clientRpc; + void *clientRpc; SCorEpSet mgmtEp; uv_mutex_t udfsMutex; - SHashObj * udfsHash; + SHashObj *udfsHash; - SArray* residentFuncs; + SArray *residentFuncs; bool printVersion; } SUdfdContext; SUdfdContext global; +struct SUdfdUvConn; +struct SUvUdfWork; + typedef struct SUdfdUvConn { uv_stream_t *client; - char * inputBuf; + char *inputBuf; int32_t inputLen; int32_t inputCap; int32_t inputTotal; + + struct SUvUdfWork *pWorkList; // head of work list } SUdfdUvConn; typedef struct SUvUdfWork { - uv_stream_t *client; + SUdfdUvConn *conn; uv_buf_t input; uv_buf_t output; + + struct SUvUdfWork *pWorkNext; } SUvUdfWork; typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY, UDF_STATE_UNLOADING } EUdfState; @@ -70,7 +77,7 @@ typedef struct SUdf { EUdfState state; uv_mutex_t lock; uv_cond_t condReady; - bool resident; + bool resident; char name[TSDB_FUNC_NAME_LEN + 1]; int8_t funcType; @@ -107,7 +114,7 @@ typedef enum EUdfdRpcReqRspType { typedef struct SUdfdRpcSendRecvInfo { EUdfdRpcReqRspType rpcType; int32_t code; - void * param; + void *param; uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; @@ -178,7 +185,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { fnInfo("setup request. seq num: %" PRId64 ", udf name: %s", request->seqNum, request->setup.udfName); SUdfSetupRequest *setup = &request->setup; int32_t code = TSDB_CODE_SUCCESS; - SUdf * udf = NULL; + SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName)); if (udfInHash) { @@ -193,7 +200,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { uv_cond_init(&udfNew->condReady); udf = udfNew; - SUdf** pUdf = &udf; + SUdf **pUdf = &udf; taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), pUdf, POINTER_BYTES); uv_mutex_unlock(&global.udfsMutex); } @@ -207,7 +214,7 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { } udf->resident = false; for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { - char* funcName = taosArrayGet(global.residentFuncs, i); + char *funcName = taosArrayGet(global.residentFuncs, i); if (strcmp(setup->udfName, funcName) == 0) { udf->resident = true; break; @@ -248,11 +255,12 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfCallRequest *call = &request->call; - fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64 , call->callType, call->udfHandle, request->seqNum); - SUdfcFuncHandle * handle = (SUdfcFuncHandle *)(call->udfHandle); - SUdf * udf = handle->udf; + fnDebug("call request. call type %d, handle: %" PRIx64 ", seq num %" PRId64, call->callType, call->udfHandle, + request->seqNum); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); + SUdf *udf = handle->udf; SUdfResponse response = {0}; - SUdfResponse * rsp = &response; + SUdfResponse *rsp = &response; SUdfCallResponse *subRsp = &rsp->callRsp; int32_t code = TSDB_CODE_SUCCESS; @@ -352,7 +360,7 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { SUdfTeardownRequest *teardown = &request->teardown; fnInfo("teardown. seq number: %" PRId64 ", handle:%" PRIx64, request->seqNum, teardown->udfHandle); SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); - SUdf * udf = handle->udf; + SUdf *udf = handle->udf; bool unloadUdf = false; int32_t code = TSDB_CODE_SUCCESS; @@ -409,17 +417,16 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); - + int32_t now = taosGetTimestampSec(); int32_t delta = abs(now - connectRsp.svrTimestamp); if (delta > 900) { msgInfo->code = TSDB_CODE_TIME_UNSYNCED; goto _return; } - - + if (connectRsp.epSet.numOfEps == 0) { - msgInfo->code = TSDB_CODE_MND_APP_ERROR; + msgInfo->code = TSDB_CODE_APP_ERROR; goto _return; } @@ -434,7 +441,7 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { goto _return; } SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); - SUdf * udf = msgInfo->param; + SUdf *udf = msgInfo->param; udf->funcType = pFuncInfo->funcType; udf->scriptType = pFuncInfo->scriptType; udf->outputType = pFuncInfo->outputType; @@ -487,7 +494,7 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { taosArrayPush(retrieveReq.pFuncNames, udfName); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); taosArrayDestroy(retrieveReq.pFuncNames); @@ -522,7 +529,7 @@ int32_t udfdConnectToMnode() { connReq.startTime = taosGetTimestampMs(); int32_t contLen = tSerializeSConnectReq(NULL, 0, &connReq); - void * pReq = rpcMallocCont(contLen); + void *pReq = rpcMallocCont(contLen); tSerializeSConnectReq(pReq, contLen, &connReq); SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo)); @@ -589,7 +596,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); - char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *mergeSuffix = "_merge"; strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); @@ -598,11 +605,13 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code, tmsg_t msgType) { - if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_BROKEN_LINK) { - if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || msgType == TDMT_SCH_MERGE_FETCH) { + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_SYN_NOT_LEADER || + code == TSDB_CODE_SYN_RESTORING || code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING) { + if (msgType == TDMT_SCH_QUERY || msgType == TDMT_SCH_MERGE_QUERY || msgType == TDMT_SCH_FETCH || + msgType == TDMT_SCH_MERGE_FETCH) { return false; - } + } return true; } else { return false; @@ -662,7 +671,7 @@ int32_t udfdOpenClientRpc() { rpcInit.parent = &global; rpcInit.rfp = udfdRpcRfp; rpcInit.compressSize = tsCompressMsgSize; - + global.clientRpc = rpcOpen(&rpcInit); if (global.clientRpc == NULL) { fnError("failed to init dnode rpc client"); @@ -683,6 +692,17 @@ void udfdOnWrite(uv_write_t *req, int status) { if (status < 0) { fnError("udfd send response error, length: %zu code: %s", work->output.len, uv_err_name(status)); } + // remove work from the connection work list + if (work->conn != NULL) { + SUvUdfWork **ppWork; + for (ppWork = &work->conn->pWorkList; *ppWork && (*ppWork != work); ppWork = &((*ppWork)->pWorkNext)) { + } + if (*ppWork == work) { + *ppWork = work->pWorkNext; + } else { + fnError("work not in conn any more"); + } + } taosMemoryFree(work->output.base); taosMemoryFree(work); taosMemoryFree(req); @@ -691,10 +711,11 @@ void udfdOnWrite(uv_write_t *req, int status) { void udfdSendResponse(uv_work_t *work, int status) { SUvUdfWork *udfWork = (SUvUdfWork *)(work->data); - uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); - write_req->data = udfWork; - uv_write(write_req, udfWork->client, &udfWork->output, 1, udfdOnWrite); - + if (udfWork->conn != NULL) { + uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t)); + write_req->data = udfWork; + uv_write(write_req, udfWork->conn->client, &udfWork->output, 1, udfdOnWrite); + } taosMemoryFree(work); } @@ -715,8 +736,8 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->len = 0; } } else if (ctx->inputTotal == -1 && ctx->inputLen < msgHeadSize) { - buf->base = ctx->inputBuf + ctx->inputLen; - buf->len = msgHeadSize - ctx->inputLen; + buf->base = ctx->inputBuf + ctx->inputLen; + buf->len = msgHeadSize - ctx->inputLen; } else { ctx->inputCap = ctx->inputTotal > ctx->inputCap ? ctx->inputTotal : ctx->inputCap; void *inputBuf = taosMemoryRealloc(ctx->inputBuf, ctx->inputCap); @@ -743,10 +764,15 @@ bool isUdfdUvMsgComplete(SUdfdUvConn *pipe) { } void udfdHandleRequest(SUdfdUvConn *conn) { - uv_work_t * work = taosMemoryMalloc(sizeof(uv_work_t)); + char *inputBuf = conn->inputBuf; + int32_t inputLen = conn->inputLen; + + uv_work_t *work = taosMemoryMalloc(sizeof(uv_work_t)); SUvUdfWork *udfWork = taosMemoryMalloc(sizeof(SUvUdfWork)); - udfWork->client = conn->client; - udfWork->input = uv_buf_init(conn->inputBuf, conn->inputLen); + udfWork->conn = conn; + udfWork->pWorkNext = conn->pWorkList; + conn->pWorkList = udfWork; + udfWork->input = uv_buf_init(inputBuf, inputLen); conn->inputBuf = NULL; conn->inputLen = 0; conn->inputCap = 0; @@ -757,13 +783,19 @@ void udfdHandleRequest(SUdfdUvConn *conn) { void udfdPipeCloseCb(uv_handle_t *pipe) { SUdfdUvConn *conn = pipe->data; + SUvUdfWork* pWork = conn->pWorkList; + while (pWork != NULL) { + pWork->conn = NULL; + pWork = pWork->pWorkNext; + } + taosMemoryFree(conn->client); taosMemoryFree(conn->inputBuf); taosMemoryFree(conn); } void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - fnDebug("udf read %zd bytes from client", nread); + fnDebug("udfd read %zd bytes from client", nread); if (nread == 0) return; SUdfdUvConn *conn = client->data; @@ -779,10 +811,10 @@ void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - fnError("Receive error %s", uv_err_name(nread)); if (nread == UV_EOF) { - // TODO check more when close + fnInfo("udfd pipe read EOF"); } else { + fnError("Receive error %s", uv_err_name(nread)); } udfdUvHandleError(conn); } @@ -798,6 +830,7 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { uv_pipe_init(global.loop, client, 0); if (uv_accept(server, (uv_stream_t *)client) == 0) { SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn)); + ctx->pWorkList = NULL; ctx->client = (uv_stream_t *)client; ctx->inputBuf = 0; ctx->inputLen = 0; @@ -890,7 +923,7 @@ static int32_t udfdUvInit() { } global.loop = loop; - if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit + if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit uv_pipe_init(global.loop, &global.ctrlPipe, 1); uv_pipe_open(&global.ctrlPipe, 0); uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); @@ -965,10 +998,10 @@ int32_t udfdInitResidentFuncs() { } global.residentFuncs = taosArrayInit(2, TSDB_FUNC_NAME_LEN); - char* pSave = tsUdfdResFuncs; - char* token; + char *pSave = tsUdfdResFuncs; + char *token; while ((token = strtok_r(pSave, ",", &pSave)) != NULL) { - char func[TSDB_FUNC_NAME_LEN+1] = {0}; + char func[TSDB_FUNC_NAME_LEN + 1] = {0}; strncpy(func, token, TSDB_FUNC_NAME_LEN); fnInfo("udfd add resident function %s", func); taosArrayPush(global.residentFuncs, func); @@ -979,10 +1012,10 @@ int32_t udfdInitResidentFuncs() { int32_t udfdDeinitResidentFuncs() { for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { - char* funcName = taosArrayGet(global.residentFuncs, i); - SUdf** udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); + char *funcName = taosArrayGet(global.residentFuncs, i); + SUdf **udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName)); if (udfInHash) { - SUdf* udf = *udfInHash; + SUdf *udf = *udfInHash; if (udf->destroyFunc) { (udf->destroyFunc)(); } @@ -1018,8 +1051,8 @@ int main(int argc, char *argv[]) { } if (udfdInitLog() != 0) { + // ignore create log failed, because this error no matter printf("failed to start since init log error\n"); - return -1; } if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 72828e1daa2be87cee76f9849d90cc47296b6509..5a86bc86780544fff62e8b5a5513da731ed56804 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -195,7 +195,7 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { } char *tv = taosMemoryCalloc(1, valLen + 1); if (tv == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tv, pData, valLen); @@ -259,7 +259,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pFilter); indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -269,7 +269,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIFParam *res = (SIFParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { indexError("no result for node, type:%d, node:%p", nodeType(node), node); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; break; @@ -300,7 +300,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); if (NULL == paramList) { - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && @@ -319,7 +319,7 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx SIF_ERR_JRET(sifInitParam(node->pRight, ¶mList[1], ctx)); // if (paramList[0].colValType == TSDB_DATA_TYPE_JSON && // ((SOperatorNode *)(node))->opType == OP_TYPE_JSON_CONTAINS) { - // return TSDB_CODE_QRY_OUT_OF_MEMORY; + // return TSDB_CODE_OUT_OF_MEMORY; //} } *params = paramList; @@ -335,7 +335,7 @@ static int32_t sifInitParamList(SIFParam **params, SNodeList *nodeList, SIFCtx * SIFParam *tParams = taosMemoryCalloc(nodeList->length, sizeof(SIFParam)); if (tParams == NULL) { indexError("failed to calloc, nodeList: %p", nodeList); - SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SListCell *cell = nodeList->pHead; @@ -464,7 +464,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); if (tm == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); @@ -722,7 +722,7 @@ static EDealRes sifWalkFunction(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -740,7 +740,7 @@ static EDealRes sifWalkLogic(SNode *pNode, void *context) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } return DEAL_RES_CONTINUE; @@ -756,7 +756,7 @@ static EDealRes sifWalkOper(SNode *pNode, void *context) { return DEAL_RES_ERROR; } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -807,7 +807,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -821,7 +821,7 @@ static int32_t sifCalculate(SNode *pNode, SIFParam *pDst) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } if (res->result != NULL) { taosArrayAddAll(pDst->result, res->result); @@ -844,7 +844,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { ctx.pRes = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { indexError("index-filter failed to taosHashInit"); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } nodesWalkExprPostOrder(pNode, sifCalcWalker, &ctx); @@ -856,7 +856,7 @@ static int32_t sifGetFltHint(SNode *pNode, SIdxFltStatus *status) { SIFParam *res = (SIFParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (res == NULL) { indexError("no valid res in hash, node:(%p), type(%d)", (void *)&pNode, nodeType(pNode)); - SIF_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SIF_ERR_RET(TSDB_CODE_APP_ERROR); } *status = res->status; sifFreeParam(res); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index e002ff9c32a42580300ea134dc764bd03283a693..b34d05d297e26f03d5af40e8b440241cf55ed714 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -269,7 +269,7 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { if (ret != 0) { taosArrayDestroy(offsets); indexError("failed to find target tablelist"); - return TSDB_CODE_TDB_FILE_CORRUPTED; + return TSDB_CODE_FILE_CORRUPTED; } } taosArrayDestroy(offsets); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 5b3e8ce5a9334e05fa184427469314482d960a74..aab018c87968b203ee79da2e4d872bd444d21082 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -295,6 +295,13 @@ static int32_t stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNod return TSDB_CODE_SUCCESS; } +static int32_t eventWindowNodeCopy(const SEventWindowNode* pSrc, SEventWindowNode* pDst) { + CLONE_NODE_FIELD(pCol); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); + return TSDB_CODE_SUCCESS; +} + static int32_t sessionWindowNodeCopy(const SSessionWindowNode* pSrc, SSessionWindowNode* pDst) { CLONE_NODE_FIELD_EX(pCol, SColumnNode*); CLONE_NODE_FIELD_EX(pGap, SValueNode*); @@ -378,6 +385,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { CLONE_NODE_FIELD(pTagIndexCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); + COPY_SCALAR_FIELD(deleteMark); COPY_SCALAR_FIELD(igExpired); CLONE_NODE_LIST_FIELD(pGroupTags); COPY_SCALAR_FIELD(groupSort); @@ -461,8 +469,11 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p CLONE_NODE_FIELD(pTspk); CLONE_NODE_FIELD(pTsEnd); CLONE_NODE_FIELD(pStateExpr); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); + COPY_SCALAR_FIELD(deleteMark); COPY_SCALAR_FIELD(igExpired); COPY_SCALAR_FIELD(windowAlgo); COPY_SCALAR_FIELD(inputTsOrder); @@ -707,6 +718,9 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_STATE_WINDOW: code = stateWindowNodeCopy((const SStateWindowNode*)pNode, (SStateWindowNode*)pDst); break; + case QUERY_NODE_EVENT_WINDOW: + code = eventWindowNodeCopy((const SEventWindowNode*)pNode, (SEventWindowNode*)pDst); + break; case QUERY_NODE_SESSION_WINDOW: code = sessionWindowNodeCopy((const SSessionWindowNode*)pNode, (SSessionWindowNode*)pDst); break; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 50e410b33918f9aeed2c317195f5556798a20566..38884a37e05400adb34e073ec8760ea447e9fdb1 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -85,6 +85,8 @@ const char* nodesNodeName(ENodeType type) { return "WhenThen"; case QUERY_NODE_CASE_WHEN: return "CaseWhen"; + case QUERY_NODE_EVENT_WINDOW: + return "EventWindow"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: @@ -221,6 +223,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiTableScan"; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: return "PhysiTableSeqScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + return "PhysiTableMergeScan"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: return "PhysiSreamScan"; case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: @@ -229,8 +233,12 @@ const char* nodesNodeName(ENodeType type) { return "PhysiBlockDistScan"; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return "PhysiLastRowScan"; - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - return "PhysiTableMergeScan"; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + return "PhysiTableCountScan"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return "PhysiMergeEventWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return "PhysiStreamEventWindow"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -817,6 +825,7 @@ static const char* jkWindowLogicPlanTspk = "Tspk"; static const char* jkWindowLogicPlanStateExpr = "StateExpr"; static const char* jkWindowLogicPlanTriggerType = "TriggerType"; static const char* jkWindowLogicPlanWatermark = "Watermark"; +static const char* jkWindowLogicPlanDeleteMark = "DeleteMark"; static int32_t logicWindowNodeToJson(const void* pObj, SJson* pJson) { const SWindowLogicNode* pNode = (const SWindowLogicNode*)pObj; @@ -858,6 +867,9 @@ static int32_t logicWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanWatermark, pNode->watermark); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanDeleteMark, pNode->deleteMark); + } return code; } @@ -902,6 +914,9 @@ static int32_t jsonToLogicWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanWatermark, &pNode->watermark); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanDeleteMark, &pNode->deleteMark); + } return code; } @@ -2002,6 +2017,7 @@ static const char* jkWindowPhysiPlanTsPk = "TsPk"; static const char* jkWindowPhysiPlanTsEnd = "TsEnd"; static const char* jkWindowPhysiPlanTriggerType = "TriggerType"; static const char* jkWindowPhysiPlanWatermark = "Watermark"; +static const char* jkWindowPhysiPlanDeleteMark = "DeleteMark"; static const char* jkWindowPhysiPlanIgnoreExpired = "IgnoreExpired"; static const char* jkWindowPhysiPlanInputTsOrder = "InputTsOrder"; static const char* jkWindowPhysiPlanOutputTsOrder = "outputTsOrder"; @@ -2029,6 +2045,9 @@ static int32_t physiWindowNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanWatermark, pNode->watermark); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanDeleteMark, pNode->deleteMark); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkWindowPhysiPlanIgnoreExpired, pNode->igExpired); } @@ -2067,6 +2086,9 @@ static int32_t jsonToPhysiWindowNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkWindowPhysiPlanWatermark, &pNode->watermark); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowPhysiPlanDeleteMark, &pNode->deleteMark); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkWindowPhysiPlanIgnoreExpired, &pNode->igExpired); } @@ -2256,6 +2278,37 @@ static int32_t jsonToPhysiStateWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowPhysiPlanStartCond = "StartCond"; +static const char* jkEventWindowPhysiPlanEndCond = "EndCond"; + +static int32_t physiEventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanEndCond, nodeToJson, pNode->pEndCond); + } + + return code; +} + +static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanEndCond, &pNode->pEndCond); + } + + return code; +} + static const char* jkPartitionPhysiPlanExprs = "Exprs"; static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys"; static const char* jkPartitionPhysiPlanTargets = "Targets"; @@ -3530,6 +3583,18 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToGroupingSetNode(const SJson* pJson, void* pObj) { + SGroupingSetNode* pNode = (SGroupingSetNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkGroupingSetType, pNode->groupingSetType, code); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkGroupingSetParameter, &pNode->pParameterList); + } + + return code; +} + static const char* jkOrderByExprExpr = "Expr"; static const char* jkOrderByExprOrder = "Order"; static const char* jkOrderByExprNullOrder = "NullOrder"; @@ -3632,6 +3697,36 @@ static int32_t jsonToSessionWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowTsPrimaryKey = "TsPrimaryKey"; +static const char* jkEventWindowStartCond = "StartCond"; +static const char* jkEventWindowEndCond = "EndCond"; + +static int32_t eventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWindowNode* pNode = (const SEventWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkEventWindowTsPrimaryKey, nodeToJson, pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowEndCond, nodeToJson, pNode->pEndCond); + } + return code; +} + +static int32_t jsonToEventWindowNode(const SJson* pJson, void* pObj) { + SEventWindowNode* pNode = (SEventWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkEventWindowTsPrimaryKey, &pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowEndCond, &pNode->pEndCond); + } + return code; +} + static const char* jkIntervalWindowInterval = "Interval"; static const char* jkIntervalWindowOffset = "Offset"; static const char* jkIntervalWindowSliding = "Sliding"; @@ -4587,6 +4682,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return whenThenNodeToJson(pObj, pJson); case QUERY_NODE_CASE_WHEN: return caseWhenNodeToJson(pObj, pJson); + case QUERY_NODE_EVENT_WINDOW: + return eventWindowNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: @@ -4646,6 +4743,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: return physiScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return physiLastRowScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: @@ -4683,6 +4781,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return physiStateWindowNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return physiEventWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return physiPartitionNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -4726,6 +4827,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToRealTableNode(pJson, pObj); case QUERY_NODE_TEMP_TABLE: return jsonToTempTableNode(pJson, pObj); + case QUERY_NODE_GROUPING_SET: + return jsonToGroupingSetNode(pJson, pObj); case QUERY_NODE_ORDER_BY_EXPR: return jsonToOrderByExprNode(pJson, pObj); case QUERY_NODE_LIMIT: @@ -4756,6 +4859,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToWhenThenNode(pJson, pObj); case QUERY_NODE_CASE_WHEN: return jsonToCaseWhenNode(pJson, pObj); + case QUERY_NODE_EVENT_WINDOW: + return jsonToEventWindowNode(pJson, pObj); case QUERY_NODE_SET_OPERATOR: return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: @@ -4800,6 +4905,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicPlan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return jsonToPhysiScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return jsonToPhysiLastRowScanNode(pJson, pObj); @@ -4839,6 +4945,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return jsonToPhysiStateWindowNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return jsonToPhysiEventWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 1e8ff8da1a9d2484604199d91732ac2cf4ee141a..cb441053cef64b6b79fd5d56c65dd01aaedd763f 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2607,6 +2607,7 @@ enum { PHY_WINDOW_CODE_TS_END, PHY_WINDOW_CODE_TRIGGER_TYPE, PHY_WINDOW_CODE_WATERMARK, + PHY_WINDOW_CODE_DELETE_MARK, PHY_WINDOW_CODE_IG_EXPIRED, PHY_WINDOW_CODE_INPUT_TS_ORDER, PHY_WINDOW_CODE_OUTPUT_TS_ORDER, @@ -2635,6 +2636,9 @@ static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeI64(pEncoder, PHY_WINDOW_CODE_WATERMARK, pNode->watermark); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_WINDOW_CODE_DELETE_MARK, pNode->deleteMark); + } if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_IG_EXPIRED, pNode->igExpired); } @@ -2679,6 +2683,9 @@ static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { case PHY_WINDOW_CODE_WATERMARK: code = tlvDecodeI64(pTlv, &pNode->watermark); break; + case PHY_WINDOW_CODE_DELETE_MARK: + code = tlvDecodeI64(pTlv, &pNode->deleteMark); + break; case PHY_WINDOW_CODE_IG_EXPIRED: code = tlvDecodeI8(pTlv, &pNode->igExpired); break; @@ -2920,6 +2927,46 @@ static int32_t msgToPhysiStateWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } +enum { PHY_EVENT_CODE_WINDOW = 1, PHY_EVENT_CODE_START_COND, PHY_EVENT_CODE_END_COND }; + +static int32_t physiEventWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_START_COND, nodeToMsg, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_END_COND, nodeToMsg, pNode->pEndCond); + } + + return code; +} + +static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_EVENT_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_EVENT_CODE_START_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartCond); + break; + case PHY_EVENT_CODE_END_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndCond); + break; + default: + break; + } + } + + return code; +} + enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -3640,6 +3687,7 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { code = physiScanNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = physiLastRowScanNodeToMsg(pObj, pEncoder); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -3690,6 +3738,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = physiStateWindowNodeToMsg(pObj, pEncoder); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = physiEventWindowNodeToMsg(pObj, pEncoder); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = physiPartitionNodeToMsg(pObj, pEncoder); break; @@ -3778,6 +3830,7 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { code = msgToPhysiScanNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: code = msgToPhysiLastRowScanNode(pDecoder, pObj); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: @@ -3828,6 +3881,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = msgToPhysiStateWindowNode(pDecoder, pObj); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = msgToPhysiEventWindowNode(pDecoder, pObj); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = msgToPhysiPartitionNode(pDecoder, pObj); break; diff --git a/source/libs/nodes/src/nodesToSQLFuncs.c b/source/libs/nodes/src/nodesToSQLFuncs.c index 9325d0288636ca7e22fe4fdd3a8e50ff90cdf0de..0181da92a9e7eeb5387a68ed18b0b20470979101 100644 --- a/source/libs/nodes/src/nodesToSQLFuncs.c +++ b/source/libs/nodes/src/nodesToSQLFuncs.c @@ -132,7 +132,7 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { char *t = nodesGetStrValueFromNode(colNode); if (NULL == t) { nodesError("fail to get str value from valueNode"); - NODES_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t tlen = strlen(t); @@ -229,5 +229,5 @@ int32_t nodesNodeToSQL(SNode *pNode, char *buf, int32_t bufSize, int32_t *len) { } nodesError("nodesNodeToSQL unknown node = %s", nodesNodeName(pNode->type)); - NODES_RET(TSDB_CODE_QRY_APP_ERROR); + NODES_RET(TSDB_CODE_APP_ERROR); } diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 106812d55f1cdd08120cb100e338b0b82244e752..ce575ede8a5dd2020c6af39394a72d6fa39cc348 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -165,6 +165,17 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = walkExpr(pEvent->pCol, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pStartCond, order, walker, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pEndCond, order, walker, pContext); + } + break; + } default: break; } @@ -329,6 +340,17 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pStartCond, order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pEndCond, order, rewriter, pContext); + } + break; + } default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index b74b7ae7204091a69054742747cb9f0f6987a466..e3210966a017ab2ba14a5c3e00f965956027daa1 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -299,6 +299,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SWhenThenNode)); case QUERY_NODE_CASE_WHEN: return makeNode(type, sizeof(SCaseWhenNode)); + case QUERY_NODE_EVENT_WINDOW: + return makeNode(type, sizeof(SEventWindowNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: @@ -494,6 +496,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SBlockDistScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return makeNode(type, sizeof(SLastRowScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + return makeNode(type, sizeof(STableCountScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -533,6 +537,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return makeNode(type, sizeof(SEventWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return makeNode(type, sizeof(SStreamEventWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -594,6 +602,13 @@ static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { nodesDestroyNode(pNode->pTsEnd); } +static void destroyPartitionPhysiNode(SPartitionPhysiNode* pNode) { + destroyPhysiNode((SPhysiNode*)pNode); + nodesDestroyList(pNode->pExprs); + nodesDestroyList(pNode->pPartitionKeys); + nodesDestroyList(pNode->pTargets); +} + static void destroyScanPhysiNode(SScanPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pScanCols); @@ -731,6 +746,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pOptions->pWatermark); nodesDestroyList(pOptions->pRollupFuncs); nodesDestroyList(pOptions->pSma); + nodesDestroyList(pOptions->pDeleteMark); break; } case QUERY_NODE_INDEX_OPTIONS: { @@ -748,22 +764,30 @@ void nodesDestroyNode(SNode* pNode) { SStreamOptions* pOptions = (SStreamOptions*)pNode; nodesDestroyNode(pOptions->pDelay); nodesDestroyNode(pOptions->pWatermark); + nodesDestroyNode(pOptions->pDeleteMark); break; } case QUERY_NODE_LEFT_VALUE: // no pointer field case QUERY_NODE_COLUMN_REF: // no pointer field break; case QUERY_NODE_WHEN_THEN: { - SWhenThenNode* pStmt = (SWhenThenNode*)pNode; - nodesDestroyNode(pStmt->pWhen); - nodesDestroyNode(pStmt->pThen); + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + nodesDestroyNode(pWhenThen->pWhen); + nodesDestroyNode(pWhenThen->pThen); break; } case QUERY_NODE_CASE_WHEN: { - SCaseWhenNode* pStmt = (SCaseWhenNode*)pNode; - nodesDestroyNode(pStmt->pCase); - nodesDestroyNode(pStmt->pElse); - nodesDestroyList(pStmt->pWhenThenList); + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; + nodesDestroyNode(pCaseWhen->pCase); + nodesDestroyNode(pCaseWhen->pElse); + nodesDestroyList(pCaseWhen->pWhenThenList); + break; + } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + nodesDestroyNode(pEvent->pCol); + nodesDestroyNode(pEvent->pStartCond); + nodesDestroyNode(pEvent->pEndCond); break; } case QUERY_NODE_SET_OPERATOR: { @@ -904,6 +928,8 @@ void nodesDestroyNode(SNode* pNode) { SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pNode; nodesDestroyNode((SNode*)pStmt->pOptions); nodesDestroyNode(pStmt->pQuery); + nodesDestroyList(pStmt->pTags); + nodesDestroyNode(pStmt->pSubtable); break; } case QUERY_NODE_DROP_STREAM_STMT: // no pointer field @@ -1019,6 +1045,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pTagIndexCond); taosArrayDestroyEx(pLogicNode->pSmaIndexes, destroySmaIndex); nodesDestroyList(pLogicNode->pGroupTags); + nodesDestroyList(pLogicNode->pTags); + nodesDestroyNode(pLogicNode->pSubtable); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -1091,6 +1119,8 @@ void nodesDestroyNode(SNode* pNode) { SPartitionLogicNode* pLogicNode = (SPartitionLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pPartitionKeys); + nodesDestroyList(pLogicNode->pTags); + nodesDestroyNode(pLogicNode->pSubtable); break; } case QUERY_NODE_LOGIC_PLAN_INDEF_ROWS_FUNC: { @@ -1123,7 +1153,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: { + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode; destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pGroupTags); @@ -1137,6 +1168,8 @@ void nodesDestroyNode(SNode* pNode) { destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pDynamicScanFuncs); nodesDestroyList(pPhyNode->pGroupTags); + nodesDestroyList(pPhyNode->pTags); + nodesDestroyNode(pPhyNode->pSubtable); break; } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { @@ -1213,13 +1246,23 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pStateKey); break; } - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { + SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; + destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pStartCond); + nodesDestroyNode(pPhyNode->pEndCond); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { + destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode); + break; + } case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: { - SPartitionPhysiNode* pPhyNode = (SPartitionPhysiNode*)pNode; - destroyPhysiNode((SPhysiNode*)pPhyNode); - nodesDestroyList(pPhyNode->pExprs); - nodesDestroyList(pPhyNode->pPartitionKeys); - nodesDestroyList(pPhyNode->pTargets); + SStreamPartitionPhysiNode* pPhyNode = (SStreamPartitionPhysiNode*)pNode; + destroyPartitionPhysiNode((SPartitionPhysiNode*)pPhyNode); + nodesDestroyList(pPhyNode->pTags); + nodesDestroyNode(pPhyNode->pSubtable); break; } case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: { @@ -1561,7 +1604,7 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { pNode->datum.p = (char*)value; break; default: - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index a6c56f3ae4239c936747fde5094ea57fe9dcba7f..c74ec9c1479b872c11daf65bf1a0df0caa093a44 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -72,7 +72,8 @@ typedef enum ETableOptionType { TABLE_OPTION_WATERMARK, TABLE_OPTION_ROLLUP, TABLE_OPTION_TTL, - TABLE_OPTION_SMA + TABLE_OPTION_SMA, + TABLE_OPTION_DELETE_MARK } ETableOptionType; typedef struct SAlterOption { @@ -115,6 +116,7 @@ SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const STok SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index e4e8cd725b76530723b28199b50a677000175654..86d98c5515311155d91057d24e99a356912bd1ac 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -141,23 +141,6 @@ int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); void insDestroyDataBlock(STableDataBlocks *pDataBlock); -typedef struct SBoundColInfo { - int16_t *pColIndex; // bound index => schema index - int32_t numOfCols; - int32_t numOfBound; -} SBoundColInfo; - -typedef struct STableDataCxt { - STableMeta *pMeta; - STSchema *pSchema; - SBoundColInfo boundColsInfo; - SArray *pValues; - SSubmitTbData *pData; - TSKEY lastTs; - bool ordered; - bool duplicateTs; -} STableDataCxt; - typedef struct SVgroupDataCxt { int32_t vgId; SSubmitReq2 *pData; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index c53d3f932020d6de7bd4d7923370a0cc5abb31a2..ce5a63f5d07d35fb7a432e878d60372b9bab6cf9 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -86,7 +86,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); -int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); +int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6521161244eccc1f12d641ac48121675bbef58f0..e01d0243a94c47d3989f674f794c720922c245c3 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -194,7 +194,7 @@ db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C). db_options(A) ::= db_options(B) TSDB_PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TSDB_PAGESIZE, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } -db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } +//db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } db_options(A) ::= db_options(B) VGROUPS NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_VGROUPS, &C); } db_options(A) ::= db_options(B) SINGLE_STABLE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_SINGLE_STABLE, &C); } db_options(A) ::= db_options(B) RETENTIONS retention_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_RETENTIONS, C); } @@ -232,7 +232,7 @@ alter_db_option(A) ::= KEEP integer_list(B). alter_db_option(A) ::= KEEP variable_list(B). { A.type = DB_OPTION_KEEP; A.pList = B; } alter_db_option(A) ::= PAGES NK_INTEGER(B). { A.type = DB_OPTION_PAGES; A.val = B; } alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.val = B; } -alter_db_option(A) ::= STRICT NK_STRING(B). { A.type = DB_OPTION_STRICT; A.val = B; } +//alter_db_option(A) ::= STRICT NK_STRING(B). { A.type = DB_OPTION_STRICT; A.val = B; } alter_db_option(A) ::= WAL_LEVEL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } alter_db_option(A) ::= STT_TRIGGER NK_INTEGER(B). { A.type = DB_OPTION_STT_TRIGGER; A.val = B; } @@ -362,6 +362,7 @@ table_options(A) ::= table_options(B) WATERMARK duration_list(C). table_options(A) ::= table_options(B) ROLLUP NK_LP rollup_func_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_ROLLUP, C); } table_options(A) ::= table_options(B) TTL NK_INTEGER(C). { A = setTableOption(pCxt, B, TABLE_OPTION_TTL, &C); } table_options(A) ::= table_options(B) SMA NK_LP col_name_list(C) NK_RP. { A = setTableOption(pCxt, B, TABLE_OPTION_SMA, C); } +table_options(A) ::= table_options(B) DELETE_MARK duration_list(C). { A = setTableOption(pCxt, B, TABLE_OPTION_DELETE_MARK, C); } alter_table_options(A) ::= alter_table_option(B). { A = createAlterTableOptions(pCxt); A = setTableOption(pCxt, A, B.type, &B.val); } alter_table_options(A) ::= alter_table_options(B) alter_table_option(C). { A = setTableOption(pCxt, B, C.type, &C.val); } @@ -475,8 +476,9 @@ func_list(A) ::= func_list(B) NK_COMMA func(C). func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } sma_stream_opt(A) ::= . { A = createStreamOptions(pCxt); } -sma_stream_opt(A) ::= stream_options(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } -sma_stream_opt(A) ::= stream_options(B) MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; } +sma_stream_opt(A) ::= sma_stream_opt(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } +sma_stream_opt(A) ::= sma_stream_opt(B) MAX_DELAY duration_literal(C). { ((SStreamOptions*)B)->pDelay = releaseRawExprNode(pCxt, C); A = B; } +sma_stream_opt(A) ::= sma_stream_opt(B) DELETE_MARK duration_literal(C). { ((SStreamOptions*)B)->pDeleteMark = releaseRawExprNode(pCxt, C); A = B; } /************************************************ create/drop topic ***************************************************/ cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_or_subquery(C). { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); } @@ -962,6 +964,8 @@ twindow_clause_opt(A) ::= twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); } +twindow_clause_opt(A) ::= + EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C). { A = createEventWindowNode(pCxt, B, C); } sliding_opt(A) ::= . { A = NULL; } sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } @@ -1065,5 +1069,5 @@ null_ordering_opt(A) ::= NULLS FIRST. null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } %fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL - FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT STRING - TIMES UPDATE VALUES VARIABLE VIEW WAL. + FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT + STRICT STRING TIMES UPDATE VALUES VARIABLE VIEW WAL. diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 40fb422c492a6b22d0bddfa46c2dc4c36ff75816..446f758ed7b696e47a9df99ee16d50e7ff3e24d8 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -605,6 +605,20 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { return (SNode*)state; } +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond) { + CHECK_PARSER_STATUS(pCxt); + SEventWindowNode* pEvent = (SEventWindowNode*)nodesMakeNode(QUERY_NODE_EVENT_WINDOW); + CHECK_OUT_OF_MEM(pEvent); + pEvent->pCol = createPrimaryKeyCol(pCxt, NULL); + if (NULL == pEvent->pCol) { + nodesDestroyNode((SNode*)pEvent); + CHECK_OUT_OF_MEM(NULL); + } + pEvent->pStartCond = pStartCond; + pEvent->pEndCond = pEndCond; + return (SNode*)pEvent; +} + SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { CHECK_PARSER_STATUS(pCxt); @@ -1124,6 +1138,9 @@ SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType case TABLE_OPTION_SMA: ((STableOptions*)pOptions)->pSma = pVal; break; + case TABLE_OPTION_DELETE_MARK: + ((STableOptions*)pOptions)->pDeleteMark = pVal; + break; default: break; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 3481eddba4895536773ed1ea22628b662c081c4e..f90a42add34c47dbbcc77ccff64a9c8b18956939 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -140,7 +140,7 @@ static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pN static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) { SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt; SName name = {0}; - int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); + int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); if (TSDB_CODE_SUCCESS == code) { code = collectMetaKeyFromInsTagsImpl(pCxt, &name); } @@ -165,7 +165,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } - if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) && + if (TSDB_CODE_SUCCESS == code && + (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 8abb31b890ad4a562977e5e22844e6a545671ecc..2ab061a4c13700558625736edf62e2316a61ce69 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -56,7 +56,7 @@ static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSche int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); @@ -101,17 +101,17 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } *tagName = taosArrayInit(8, TSDB_COL_NAME_LEN); if (!*tagName) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -158,14 +158,74 @@ end: return code; } -int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta){ + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq *pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), + pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } + + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt){ + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void *data, int32_t index){ + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if (kv->type == TSDB_DATA_TYPE_NCHAR){ + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if(kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t *)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - SBoundColInfo bindTags = {0}; - SVCreateTbReq *pCreateTblReq = NULL; - SArray* tagName = NULL; + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SBoundColInfo bindTags = {0}; + SVCreateTbReq* pCreateTblReq = NULL; + SArray* tagName = NULL; insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); @@ -174,7 +234,7 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col goto end; } - STag* pTag = NULL; + STag* pTag = NULL; ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); if (ret != TSDB_CODE_SUCCESS) { @@ -186,15 +246,29 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col ret = TSDB_CODE_OUT_OF_MEMORY; goto end; } - insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags, ttl); + insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags, + ttl); pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + if(dataFormat){ + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } + STableDataCxt* pTableCxt = NULL; - ret = insGetTableDataCxt(((SVnodeModifOpStmt *)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), - pTableMeta, &pCreateTblReq, &pTableCxt, false); + ret = insGetTableDataCxt(((SVnodeModifOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; @@ -220,25 +294,22 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col } for (int32_t r = 0; r < rowNum; ++r) { - void* rowData = taosArrayGetP(cols, r); + void* rowData = taosArrayGetP(cols, r); // 1. set the parsed value from sql string for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); - SSmlKv* kv = NULL; - if (!format){ - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; - } - - if (kv == NULL) { + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { continue; } + SSmlKv *kv = *(SSmlKv **)p; + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); } - if (kv->type == TSDB_DATA_TYPE_NCHAR){ + if (kv->type == TSDB_DATA_TYPE_NCHAR) { int32_t len = 0; char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); if (NULL == pUcs4) { @@ -256,9 +327,9 @@ int32_t smlBindData(SQuery* query, SArray* tags, SArray* colsSchema, SArray* col } pVal->value.pData = pUcs4; pVal->value.nData = len; - } else if(kv->type == TSDB_DATA_TYPE_BINARY) { - pVal->value.nData = kv->length; - pVal->value.pData = (uint8_t *)kv->value; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; } else { memcpy(&pVal->value.val, &(kv->value), kv->length); } @@ -282,31 +353,30 @@ end: } SQuery* smlInitHandle() { - SQuery *pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { uError("create pQuery error"); - return NULL; + return NULL; } pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = false; pQuery->msgType = TDMT_VND_SUBMIT; - SVnodeModifOpStmt *stmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifOpStmt* stmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (NULL == stmt) { uError("create SVnodeModifOpStmt error"); qDestroyQuery(pQuery); return NULL; } - stmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - stmt->pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); stmt->freeHashFunc = insDestroyTableDataCxtHashMap; stmt->freeArrayFunc = insDestroyVgroupDataCxtList; - pQuery->pRoot = (SNode *)stmt; + pQuery->pRoot = (SNode*)stmt; return pQuery; } -int32_t smlBuildOutput(SQuery * handle, SHashObj* pVgHash) { - SVnodeModifOpStmt *pStmt = (SVnodeModifOpStmt*)(handle)->pRoot; +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(handle)->pRoot; // merge according to vgId int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a0a975b225991246de6564a417d5390a47f42b68..742d7cd806d7b437ddba5478faf4c99ae7283f56 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -44,6 +44,7 @@ typedef struct SInsertParseContext { SBoundColInfo tags; // for stmt bool missCache; bool usingDuplicateTable; + bool forceUpdate; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -797,14 +798,69 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, boo return code; } +static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { + SVgroupInfo vg; + int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStmt->pTableMeta) { + code = taosHashPut(pStmt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); + } + *pMissCache = (NULL == pStmt->pTableMeta); + } + return code; +} + +static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { + SParseContext* pComCxt = pCxt->pComCxt; + int32_t code = TSDB_CODE_SUCCESS; + if (pComCxt->async) { + code = getTableMetaAndVgroupImpl(pComCxt, pStmt, pMissCache); + } else { + code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, pMissCache); + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); + } + } + return code; +} + +static int32_t collectUseTable(const SName* pName, SHashObj* pTable) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); +} + +static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + return taosHashPut(pDbs, dbFName, strlen(dbFName), dbFName, sizeof(dbFName)); +} + static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (pCxt->forceUpdate) { + pCxt->missCache = true; + return TSDB_CODE_SUCCESS; + } + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); +#if 0 if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->targetTableName, false, &pStmt->pTableMeta, &pCxt->missCache); } if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableVgroup(pCxt->pComCxt, pStmt, false, &pCxt->missCache); } +#else + if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { + code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); + } +#endif + if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { + code = collectUseDatabase(&pStmt->targetTableName, pStmt->pDbFNameHashObj); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(&pStmt->targetTableName, pStmt->pTableNameHashObj); + } + } return code; } @@ -813,6 +869,11 @@ static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpSt } static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { + if (pCxt->forceUpdate) { + pCxt->missCache = true; + return TSDB_CODE_SUCCESS; + } + int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); @@ -820,6 +881,12 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableVgroup(pCxt->pComCxt, pStmt, true, &pCxt->missCache); } + if (TSDB_CODE_SUCCESS == code && !pCxt->pComCxt->async) { + code = collectUseDatabase(&pStmt->usingTableName, pStmt->pDbFNameHashObj); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(&pStmt->usingTableName, pStmt->pTableNameHashObj); + } + } return code; } @@ -887,10 +954,10 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { - return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, false); + return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); } - + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); if (pStmt->usingTableProcessing) { @@ -928,7 +995,7 @@ int32_t initTableColSubmitData(STableDataCxt* pTableCxt) { } for (int32_t i = 0; i < pTableCxt->boundColsInfo.numOfBound; ++i) { - SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; + SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; SColData* pCol = taosArrayReserve(pTableCxt->pData->aCol, 1); if (NULL == pCol) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1065,7 +1132,8 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - pVal->value.val = *(int64_t*)&dv; + float f = dv; + memcpy(&pVal->value.val, &f, sizeof(f)); break; } case TSDB_DATA_TYPE_DOUBLE: { @@ -1085,7 +1153,11 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - pVal->value.pData = pToken->z; + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); pVal->value.nData = pToken->n; break; } @@ -1112,7 +1184,11 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - pVal->value.pData = pToken->z; + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); pVal->value.nData = pToken->n; break; } @@ -1152,6 +1228,16 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo return code; } +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (IS_VAR_DATA_TYPE(pCol->type)) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; @@ -1205,6 +1291,8 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC *pGotRow = true; } + clearColValArray(pTableCxt->pValues); + return code; } @@ -1267,6 +1355,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; + bool firstLine = (pStmt->fileProcessing == false); pStmt->fileProcessing = false; while (TSDB_CODE_SUCCESS == code && (readLen = taosGetLineFile(pStmt->fp, &pLine)) != -1) { if (('\r' == pLine[readLen - 1]) || ('\n' == pLine[readLen - 1])) { @@ -1274,6 +1363,7 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (readLen == 0) { + firstLine = false; continue; } @@ -1282,7 +1372,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken token; strtolower(pLine, pLine); const char* pRow = pLine; + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); + if (code && firstLine) { + firstLine = false; + code = 0; + continue; + } } if (TSDB_CODE_SUCCESS == code && gotRow) { @@ -1293,6 +1389,8 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, pStmt->fileProcessing = true; break; } + + firstLine = false; } taosMemoryFree(pLine); @@ -1435,13 +1533,14 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); SStmtCallback* pStmtCb = pCxt->pComCxt->pStmtCb; - int32_t code = (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, &pStmt->targetTableName, pStmt->usingTableProcessing, - pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, pStmt->usingTableName.tname); + int32_t code = (*pStmtCb->setInfoFn)(pStmtCb->pStmt, pStmt->pTableMeta, tags, &pStmt->targetTableName, + pStmt->usingTableProcessing, pStmt->pVgroupsHashObj, pStmt->pTableBlockHashObj, + pStmt->usingTableName.tname); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pStmt->pVgroupsHashObj = NULL; @@ -1674,16 +1773,25 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR static int32_t setRefreshMate(SQuery* pQuery) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; - SName* pTable = taosHashIterate(pStmt->pTableNameHashObj, NULL); - while (NULL != pTable) { - taosArrayPush(pQuery->pTableList, pTable); - pTable = taosHashIterate(pStmt->pTableNameHashObj, pTable); + + if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { + taosArrayDestroy(pQuery->pTableList); + pQuery->pTableList = taosArrayInit(taosHashGetSize(pStmt->pTableNameHashObj), sizeof(SName)); + SName* pTable = taosHashIterate(pStmt->pTableNameHashObj, NULL); + while (NULL != pTable) { + taosArrayPush(pQuery->pTableList, pTable); + pTable = taosHashIterate(pStmt->pTableNameHashObj, pTable); + } } - char* pDb = taosHashIterate(pStmt->pDbFNameHashObj, NULL); - while (NULL != pDb) { - taosArrayPush(pQuery->pDbList, pDb); - pDb = taosHashIterate(pStmt->pDbFNameHashObj, pDb); + if (taosHashGetSize(pStmt->pDbFNameHashObj) > 0) { + taosArrayDestroy(pQuery->pDbList); + pQuery->pDbList = taosArrayInit(taosHashGetSize(pStmt->pDbFNameHashObj), TSDB_DB_FNAME_LEN); + char* pDb = taosHashIterate(pStmt->pDbFNameHashObj, NULL); + while (NULL != pDb) { + taosArrayPush(pQuery->pDbList, pDb); + pDb = taosHashIterate(pStmt->pDbFNameHashObj, pDb); + } } return TSDB_CODE_SUCCESS; @@ -1797,28 +1905,28 @@ static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStm } static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { + SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; if (pCxt->missCache) { - parserDebug("0x%" PRIx64 " %d rows have been inserted before cache miss", pCxt->pComCxt->requestId, - ((SVnodeModifOpStmt*)pQuery->pRoot)->totalRowsNum); + parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted before cache miss", pCxt->pComCxt->requestId, + pStmt->totalRowsNum, pStmt->totalTbNum); pQuery->execStage = QUERY_EXEC_STAGE_PARSE; - return buildInsertCatalogReq(pCxt, (SVnodeModifOpStmt*)pQuery->pRoot, pCatalogReq); + return buildInsertCatalogReq(pCxt, pStmt, pCatalogReq); } - parserDebug("0x%" PRIx64 " %d rows have been inserted", pCxt->pComCxt->requestId, - ((SVnodeModifOpStmt*)pQuery->pRoot)->totalRowsNum); + parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted", pCxt->pComCxt->requestId, pStmt->totalRowsNum, + pStmt->totalTbNum); pQuery->execStage = QUERY_EXEC_STAGE_SCHEDULE; return TSDB_CODE_SUCCESS; } int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatalogReq, const SMetaData* pMetaData) { - SInsertParseContext context = { - .pComCxt = pCxt, - .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, - .missCache = false, - .usingDuplicateTable = false, - }; + SInsertParseContext context = {.pComCxt = pCxt, + .msg = {.buf = pCxt->pMsg, .len = pCxt->msgLen}, + .missCache = false, + .usingDuplicateTable = false, + .forceUpdate = (NULL != pCatalogReq ? pCatalogReq->forceUpdate : false)}; int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 517cc24ff69800d549df3d55a719030a9d613406..3aebf89f4196670a0c958451bc2a02e482473d16 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -22,6 +22,36 @@ #include "ttime.h" #include "ttypes.h" +typedef struct SKvParam { + int16_t pos; + SArray* pTagVals; + SSchema* schema; + char buf[TSDB_MAX_TAGS_LEN]; +} SKvParam; + +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData **pData) { + *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == *pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSubmitTbData *pNew = *pData; + + *pNew = *pDataBlock->pData; + + cloneSVreateTbReq(pDataBlock->pData->pCreateTbReq, &pNew->pCreateTbReq); + pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); + + int32_t colNum = taosArrayGetSize(pNew->aCol); + for (int32_t i = 0; i < colNum; ++i) { + SColData *pCol = (SColData*)taosArrayGet(pNew->aCol, i); + tColDataDeepClear(pCol); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { int32_t code = TSDB_CODE_SUCCESS; SArray* pVgDataBlocks = NULL; @@ -48,7 +78,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch int32_t code = TSDB_CODE_SUCCESS; SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); @@ -134,6 +164,14 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } + if (NULL == pDataBlock->pData->pCreateTbReq) { + pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pDataBlock->pData->pCreateTbReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); end: @@ -174,7 +212,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* continue; } - if (!taosMbsToUcs4(src->buffer + src->buffer_length * i, src->length[i], (TdUcs4*)(dst->buffer + dst->buffer_length * i), dst->buffer_length, &output)) { + if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { if (errno == E2BIG) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } @@ -285,7 +323,7 @@ _return: int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { - *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -312,7 +350,7 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { @@ -350,7 +388,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel return TSDB_CODE_SUCCESS; } -int32_t qResetStmtDataBlock(void* block, bool deepClear) { +int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { STableDataCxt* pBlock = (STableDataCxt*)block; int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); @@ -366,7 +404,7 @@ int32_t qResetStmtDataBlock(void* block, bool deepClear) { return TSDB_CODE_SUCCESS; } -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc, bool reset) { +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset) { int32_t code = 0; *pDst = taosMemoryCalloc(1, sizeof(STableDataCxt)); @@ -429,7 +467,7 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc, bool reset) { return code; } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, bool rebuildCreateTb) { int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); if (code) { return code; @@ -439,14 +477,25 @@ int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgI if (pBlock->pMeta) { pBlock->pMeta->uid = uid; pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; + } + + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; + + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } } return TSDB_CODE_SUCCESS; } -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } -void qDestroyStmtDataBlock(void* pBlock) { +void qDestroyStmtDataBlock(STableDataCxt* pBlock) { if (pBlock == NULL) { return; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 3762e3c55012f31265cdb34207f65aa8cf774206..39c027f306fb3e1694ca483cebcbac0c39772115 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -205,12 +205,11 @@ void qDestroyBoundColInfo(void* pInfo) { taosMemoryFreeClear(pBoundInfo->pColIndex); } - -static int32_t createTableDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, +static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } dataBuf->nAllocSize = (uint32_t)defaultSize; @@ -224,7 +223,7 @@ static int32_t createTableDataBlock(size_t defaultSize, int32_t rowSize, int32_t dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); if (dataBuf->pData == NULL) { taosMemoryFreeClear(dataBuf); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); @@ -261,7 +260,7 @@ int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbR memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); } else { pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -362,7 +361,7 @@ static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -530,7 +529,7 @@ static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* p if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); if (tmp == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; @@ -680,7 +679,7 @@ int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { insDestroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(dataBuf->pData); taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -733,7 +732,7 @@ int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { } else { // do nothing, if allocate more memory failed pDataBlock->nAllocSize = nAllocSizeOld; - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -950,13 +949,13 @@ int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray* size_t numOfVg = taosArrayGetSize(pVgDataBlocks); *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); if (NULL == *pDataBlocks) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (size_t i = 0; i < numOfVg; ++i) { STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); dst->numOfTables = src->numOfTables; @@ -1013,7 +1012,8 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void destroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } -static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, bool colMode) { +static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, + bool colMode) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1075,6 +1075,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat if (TSDB_CODE_SUCCESS == code) { *pOutput = pTableCxt; + qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId); } else { taosMemoryFree(pTableCxt); } @@ -1082,11 +1083,20 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat return code; } +static void resetColValues(SArray* pValues) { + int32_t num = taosArrayGetSize(pValues); + for (int32_t i = 0; i < num; ++i) { + SColVal* pVal = taosArrayGet(pValues, i); + pVal->flag = CV_FLAG_NONE; + } +} + int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; + resetColValues((*pTableCxt)->pValues); return TSDB_CODE_SUCCESS; } int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); @@ -1125,6 +1135,7 @@ void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { } tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pVgCxt->pData); taosMemoryFree(pVgCxt); } @@ -1182,6 +1193,8 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); taosMemoryFreeClear(pTableCxt->pData); + qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); + return TSDB_CODE_SUCCESS; } @@ -1226,23 +1239,33 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { if (NULL == pVgroupHash || NULL == pVgroupList) { taosHashCleanup(pVgroupHash); taosArrayDestroy(pVgroupList); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; - bool colFormat = false; - + bool colFormat = false; + void* p = taosHashIterate(pTableHash, NULL); if (p) { STableDataCxt* pTableCxt = *(STableDataCxt**)p; colFormat = (0 != (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)); } - + while (TSDB_CODE_SUCCESS == code && NULL != p) { STableDataCxt* pTableCxt = *(STableDataCxt**)p; if (colFormat) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); + if (pCol->nVal <= 0) { + p = taosHashIterate(pTableHash, p); + continue; + } + + if (pTableCxt->pData->pCreateTbReq) { + pTableCxt->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + } + taosArraySort(pTableCxt->pData->aCol, insColDataComp); - + tColDataSortMerge(pTableCxt->pData->aCol); } else { if (!pTableCxt->ordered) { @@ -1252,7 +1275,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); } } - + if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; @@ -1275,7 +1298,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { if (TSDB_CODE_SUCCESS == code) { *pVgDataBlocks = pVgroupList; } else { - taosArrayDestroy(pVgroupList); + insDestroyVgroupDataCxtList(pVgroupList); } return code; @@ -1291,7 +1314,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin len += sizeof(SMsgHead); pBuf = taosMemoryMalloc(len); if (NULL == pBuf) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } ((SMsgHead*)pBuf)->vgId = htonl(vgId); ((SMsgHead*)pBuf)->contLen = htonl(len); @@ -1319,7 +1342,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, size_t numOfVg = taosArrayGetSize(pVgDataCxtList); SArray* pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); if (NULL == pDataBlocks) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } int32_t code = TSDB_CODE_SUCCESS; @@ -1327,7 +1350,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, SVgroupDataCxt* src = taosArrayGetP(pVgDataCxtList, i); SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { - code = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; } if (TSDB_CODE_SUCCESS == code) { dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); @@ -1337,7 +1360,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } if (TSDB_CODE_SUCCESS == code) { - code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_TSC_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); } } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 71231731610a341e254ba15a81677293bb8c9135..1f9e4e9ab1b58478d5e57b7e3aaff2e99d884c99 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -74,6 +74,7 @@ static SKeyword keywordTable[] = { {"DATABASES", TK_DATABASES}, {"DBS", TK_DBS}, {"DELETE", TK_DELETE}, + {"DELETE_MARK", TK_DELETE_MARK}, {"DESC", TK_DESC}, {"DESCRIBE", TK_DESCRIBE}, {"DISTINCT", TK_DISTINCT}, @@ -89,6 +90,7 @@ static SKeyword keywordTable[] = { {"EXISTS", TK_EXISTS}, {"EXPIRED", TK_EXPIRED}, {"EXPLAIN", TK_EXPLAIN}, + {"EVENT_WINDOW", TK_EVENT_WINDOW}, {"EVERY", TK_EVERY}, {"FILE", TK_FILE}, {"FILL", TK_FILL}, @@ -194,15 +196,16 @@ static SKeyword keywordTable[] = { {"SNODES", TK_SNODES}, {"SOFFSET", TK_SOFFSET}, {"SPLIT", TK_SPLIT}, - {"STT_TRIGGER", TK_STT_TRIGGER}, {"STABLE", TK_STABLE}, {"STABLES", TK_STABLES}, + {"START", TK_START}, {"STATE", TK_STATE}, {"STATE_WINDOW", TK_STATE_WINDOW}, {"STORAGE", TK_STORAGE}, {"STREAM", TK_STREAM}, {"STREAMS", TK_STREAMS}, {"STRICT", TK_STRICT}, + {"STT_TRIGGER", TK_STT_TRIGGER}, {"SUBSCRIBE", TK_SUBSCRIBE}, {"SUBSCRIPTIONS", TK_SUBSCRIPTIONS}, {"SUBTABLE", TK_SUBTABLE}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 15682a2cfe68768526bfce06edd50f0abbe86ce9..3f66a4f8de10b24af36a9dd12da16ed437e6fb07 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -47,6 +47,7 @@ typedef struct STranslateContext { SParseMetaCache* pMetaCache; bool createStream; bool stableQuery; + bool showRewrite; } STranslateContext; typedef struct SFullDatabaseName { @@ -2209,22 +2210,28 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { } static bool sysTableFromVnode(const char* pTable) { - return (0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); + return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } -static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, - SArray** pVgroupList) { +static int32_t getVnodeSysTableVgroupListImpl(STranslateContext* pCxt, SName* pTargetName, SName* pName, + SArray** pVgroupList) { if (0 == pTargetName->type) { return getDBVgInfoImpl(pCxt, pName, pVgroupList); } + if (0 == strcmp(pTargetName->dbname, TSDB_INFORMATION_SCHEMA_DB) || + 0 == strcmp(pTargetName->dbname, TSDB_PERFORMANCE_SCHEMA_DB)) { + pTargetName->type = 0; + return TSDB_CODE_SUCCESS; + } + if (TSDB_DB_NAME_T == pTargetName->type) { int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); - if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || - TSDB_CODE_MND_DB_IN_DROPPING == code) { + if (!pCxt->showRewrite && (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || + TSDB_CODE_MND_DB_IN_DROPPING == code)) { + // system table query should not report errors code = TSDB_CODE_SUCCESS; } return code; @@ -2241,50 +2248,44 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge } } else if (TSDB_CODE_MND_DB_NOT_EXIST == code || TSDB_CODE_MND_DB_IN_CREATING == code || TSDB_CODE_MND_DB_IN_DROPPING == code) { + // system table query should not report errors code = TSDB_CODE_SUCCESS; } return code; } -static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { +static int32_t getVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgs, bool* pHasUserDbCond) { if (!isSelectStmt(pCxt->pCurrStmt)) { return TSDB_CODE_SUCCESS; } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SName targetName = {0}; - int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); + int32_t code = getVnodeSysTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); if (TSDB_CODE_SUCCESS == code) { - code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); + code = getVnodeSysTableVgroupListImpl(pCxt, &targetName, pName, pVgs); } + *pHasUserDbCond = (0 != targetName.type && taosArrayGetSize(*pVgs) > 0); return code; } static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* vgroupList = NULL; - if (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS)) { - code = getTagsTableVgroupList(pCxt, pName, &vgroupList); - } else if ('\0' != pRealTable->qualDbName[0]) { - if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { - code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); - } - } else { - code = getDBVgInfoImpl(pCxt, pName, &vgroupList); - } + bool hasUserDbCond = false; + SArray* pVgs = NULL; + int32_t code = getVnodeSysTableVgroupList(pCxt, pName, &pVgs, &hasUserDbCond); if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && - isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(vgroupList)) { + isSelectStmt(pCxt->pCurrStmt) && 0 == taosArrayGetSize(pVgs)) { ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES)) { - code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) { + code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } if (TSDB_CODE_SUCCESS == code) { - code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); + code = toVgroupsInfo(pVgs, &pRealTable->pVgroupList); } - taosArrayDestroy(vgroupList); + taosArrayDestroy(pVgs); return code; } @@ -2309,30 +2310,39 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea } } +static int32_t setSuperTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + SArray* vgroupList = NULL; + int32_t code = getDBVgInfoImpl(pCxt, pName, &vgroupList); + if (TSDB_CODE_SUCCESS == code) { + code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); + } + taosArrayDestroy(vgroupList); + return code; +} + +static int32_t setNormalTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pRealTable->pVgroupList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pRealTable->pVgroupList->numOfVgroups = 1; + return getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups); +} + static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { if (pCxt->pParseCxt->topicQuery) { return TSDB_CODE_SUCCESS; } - int32_t code = TSDB_CODE_SUCCESS; if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { - SArray* vgroupList = NULL; - code = getDBVgInfoImpl(pCxt, pName, &vgroupList); - if (TSDB_CODE_SUCCESS == code) { - code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); - } - taosArrayDestroy(vgroupList); - } else if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { - code = setSysTableVgroupList(pCxt, pName, pRealTable); - } else { - pRealTable->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); - if (NULL == pRealTable->pVgroupList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pRealTable->pVgroupList->numOfVgroups = 1; - code = getTableHashVgroupImpl(pCxt, pName, pRealTable->pVgroupList->vgroups); + return setSuperTableVgroupList(pCxt, pName, pRealTable); } - return code; + + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { + return setSysTableVgroupList(pCxt, pName, pRealTable); + } + + return setNormalTableVgroupList(pCxt, pName, pRealTable); } static uint8_t getStmtPrecision(SNode* pStmt) { @@ -2366,7 +2376,6 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLE_DISTRIBUTED) && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); @@ -2828,7 +2837,7 @@ static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { return TSDB_CODE_SUCCESS; } -static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjectionList) { +static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjectionList, SHashObj** pOutput) { SHashObj* pUserAliasSet = taosHashInit(LIST_LENGTH(pProjectionList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SNode* pProject = NULL; @@ -2840,13 +2849,17 @@ static int32_t checkProjectAlias(STranslateContext* pCxt, SNodeList* pProjection } taosHashPut(pUserAliasSet, pExpr->userAlias, strlen(pExpr->userAlias), &pExpr, POINTER_BYTES); } - taosHashCleanup(pUserAliasSet); + if (NULL == pOutput) { + taosHashCleanup(pUserAliasSet); + } else { + *pOutput = pUserAliasSet; + } return TSDB_CODE_SUCCESS; } static int32_t translateProjectionList(STranslateContext* pCxt, SSelectStmt* pSelect) { if (pSelect->isSubquery) { - return checkProjectAlias(pCxt, pSelect->pProjectionList); + return checkProjectAlias(pCxt, pSelect->pProjectionList, NULL); } return rewriteProjectAlias(pSelect->pProjectionList); } @@ -3134,6 +3147,15 @@ static int32_t translateSessionWindow(STranslateContext* pCxt, SSelectStmt* pSel return TSDB_CODE_SUCCESS; } +static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, + "EVENT_WINDOW requires valid time series input"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: @@ -3142,6 +3164,8 @@ static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSe return translateSessionWindow(pCxt, pSelect); case QUERY_NODE_INTERVAL_WINDOW: return translateIntervalWindow(pCxt, pSelect); + case QUERY_NODE_EVENT_WINDOW: + return translateEventWindow(pCxt, pSelect); default: break; } @@ -3795,10 +3819,11 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } -static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int32_t val, int32_t minVal, - int32_t maxVal) { +static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, + int64_t maxVal) { if (val >= 0 && (val < minVal || val > maxVal)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %d valid range: [%d, %d]", pName, val, + return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, + "Invalid option %s: %" PRId64 " valid range: [%" PRId64 ", %" PRId64 "]", pName, val, minVal, maxVal); } return TSDB_CODE_SUCCESS; @@ -3809,8 +3834,8 @@ static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, in return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal); } -static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t minVal, - int32_t maxVal) { +static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, + int64_t maxVal) { return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal); } @@ -3869,12 +3894,16 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt pOptions->keep[2] = getBigintFromValueNode((SValueNode*)nodesListGetNode(pOptions->pKeep, 2)); } + int64_t tsdbMaxKeep = TSDB_MAX_KEEP; + if (pOptions->precision == TSDB_TIME_PRECISION_NANO) { + tsdbMaxKeep = TSDB_MAX_KEEP_NS; + } + if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || - pOptions->keep[0] > TSDB_MAX_KEEP || pOptions->keep[1] > TSDB_MAX_KEEP || pOptions->keep[2] > TSDB_MAX_KEEP) { + pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", - pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, - TSDB_MAX_KEEP); + pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, tsdbMaxKeep); } if (!((pOptions->keep[0] <= pOptions->keep[1]) && (pOptions->keep[1] <= pOptions->keep[2]))) { @@ -4026,7 +4055,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName TSDB_MAX_MINROWS_FBLOCK); } if (TSDB_CODE_SUCCESS == code) { - code = checkDbKeepOption(pCxt, pOptions); + code = checkDbPrecisionOption(pCxt, pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbKeepOption(pCxt, pOptions); // use precision } if (TSDB_CODE_SUCCESS == code) { code = checkDbRangeOption(pCxt, "pages", pOptions->pages, TSDB_MIN_PAGES_PER_VNODE, TSDB_MAX_PAGES_PER_VNODE); @@ -4039,9 +4071,6 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, TSDB_MAX_TSDB_PAGESIZE); } - if (TSDB_CODE_SUCCESS == code) { - code = checkDbPrecisionOption(pCxt, pOptions); - } if (TSDB_CODE_SUCCESS == code) { code = checkDbEnumOption(pCxt, "replications", pOptions->replica, TSDB_MIN_DB_REPLICA, TSDB_MAX_DB_REPLICA); } @@ -4454,6 +4483,37 @@ static int32_t checkTableWatermarkOption(STranslateContext* pCxt, STableOptions* return code; } +static int32_t getTableDeleteMarkOption(STranslateContext* pCxt, SValueNode* pVal, int64_t* pMaxDelay) { + return getTableDelayOrWatermarkOption(pCxt, "delete_mark", TSDB_MIN_ROLLUP_DELETE_MARK, TSDB_MAX_ROLLUP_DELETE_MARK, + pVal, pMaxDelay); +} + +static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions* pOptions, bool createStable, + SDbCfgInfo* pDbCfg) { + if (NULL == pOptions->pDeleteMark) { + return TSDB_CODE_SUCCESS; + } + + if (!createStable || NULL == pDbCfg->pRetensions) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION, + "Invalid option delete_mark: Only supported for create super table in databases " + "configured with the 'RETENTIONS' option"); + } + + if (LIST_LENGTH(pOptions->pDeleteMark) > 2) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TABLE_OPTION, "Invalid option delete_mark"); + } + + int32_t code = + getTableDeleteMarkOption(pCxt, (SValueNode*)nodesListGetNode(pOptions->pDeleteMark, 0), &pOptions->deleteMark1); + if (TSDB_CODE_SUCCESS == code && 2 == LIST_LENGTH(pOptions->pDeleteMark)) { + code = + getTableDeleteMarkOption(pCxt, (SValueNode*)nodesListGetNode(pOptions->pDeleteMark, 1), &pOptions->deleteMark2); + } + + return code; +} + static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, @@ -4473,6 +4533,9 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableWatermarkOption(pCxt, pStmt->pOptions, createStable, &dbCfg); } + if (TSDB_CODE_SUCCESS == code) { + code = checkTableDeleteMarkOption(pCxt, pStmt->pOptions, createStable, &dbCfg); + } if (TSDB_CODE_SUCCESS == code) { code = checkTableRollupOption(pCxt, pStmt->pOptions->pRollupFuncs, createStable, &dbCfg); } @@ -4740,6 +4803,8 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm pReq->delay2 = pStmt->pOptions->maxDelay2; pReq->watermark1 = pStmt->pOptions->watermark1; pReq->watermark2 = pStmt->pOptions->watermark2; + pReq->deleteMark1 = pStmt->pOptions->deleteMark1; + pReq->deleteMark2 = pStmt->pOptions->deleteMark2; pReq->colVer = 1; pReq->tagVer = 1; pReq->source = TD_REQ_FROM_APP; @@ -5135,20 +5200,34 @@ static int32_t buildCreateSmaReq(STranslateContext* pCxt, SCreateIndexStmt* pStm (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pReq->interval); pReq->slidingUnit = (NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pReq->intervalUnit); + + int32_t code = TSDB_CODE_SUCCESS; if (NULL != pStmt->pOptions->pStreamOptions) { SStreamOptions* pStreamOpt = (SStreamOptions*)pStmt->pOptions->pStreamOptions; - pReq->maxDelay = (NULL != pStreamOpt->pDelay ? ((SValueNode*)pStreamOpt->pDelay)->datum.i : -1); - pReq->watermark = (NULL != pStreamOpt->pWatermark ? ((SValueNode*)pStreamOpt->pWatermark)->datum.i - : TSDB_DEFAULT_ROLLUP_WATERMARK); - if (pReq->watermark < TSDB_MIN_ROLLUP_WATERMARK) { - pReq->watermark = TSDB_MIN_ROLLUP_WATERMARK; + if (NULL != pStreamOpt->pDelay) { + code = getTableMaxDelayOption(pCxt, (SValueNode*)pStreamOpt->pDelay, &pReq->maxDelay); + } else { + pReq->maxDelay = -1; } - if (pReq->watermark > TSDB_MAX_ROLLUP_WATERMARK) { - pReq->watermark = TSDB_MAX_ROLLUP_WATERMARK; + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStreamOpt->pWatermark) { + code = getTableWatermarkOption(pCxt, (SValueNode*)pStreamOpt->pWatermark, &pReq->watermark); + } else { + pReq->watermark = TSDB_DEFAULT_ROLLUP_WATERMARK; + } + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStreamOpt->pDeleteMark) { + code = getTableDeleteMarkOption(pCxt, (SValueNode*)pStreamOpt->pDeleteMark, &pReq->deleteMark); + } else { + pReq->deleteMark = TSDB_DEFAULT_ROLLUP_DELETE_MARK; + } } } - int32_t code = getSmaIndexDstVgId(pCxt, pStmt->dbName, pStmt->tableName, &pReq->dstVgId); + if (TSDB_CODE_SUCCESS == code) { + code = getSmaIndexDstVgId(pCxt, pStmt->dbName, pStmt->tableName, &pReq->dstVgId); + } if (TSDB_CODE_SUCCESS == code) { code = getSmaIndexSql(pCxt, &pReq->sql, &pReq->sqlLen); } @@ -5176,16 +5255,6 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS code = doTranslateValue(pCxt, (SValueNode*)pStmt->pOptions->pSliding); } - if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pOptions->pStreamOptions) { - SStreamOptions* pStreamOpt = (SStreamOptions*)pStmt->pOptions->pStreamOptions; - if (NULL != pStreamOpt->pWatermark) { - code = doTranslateValue(pCxt, (SValueNode*)pStreamOpt->pWatermark); - } - if (TSDB_CODE_SUCCESS == code && NULL != pStreamOpt->pDelay) { - code = doTranslateValue(pCxt, (SValueNode*)pStreamOpt->pDelay); - } - } - return code; } @@ -5449,9 +5518,24 @@ static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { tNameGetFullDbName(&name, pDbFName); } -static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { - SSelectStmt* pSelect = (SSelectStmt*)pStmt; - SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); +static void getStreamQueryFirstProjectAliasName(SHashObj* pUserAliasSet, char* aliasName, int32_t len) { + if (NULL == taosHashGet(pUserAliasSet, "_wstart", strlen("_wstart"))) { + snprintf(aliasName, len, "%s", "_wstart"); + return; + } + if (NULL == taosHashGet(pUserAliasSet, "ts", strlen("ts"))) { + snprintf(aliasName, len, "%s", "ts"); + return; + } + do { + taosRandStr(aliasName, len - 1); + aliasName[len - 1] = '\0'; + } while (NULL != taosHashGet(pUserAliasSet, aliasName, strlen(aliasName))); + return; +} + +static int32_t addWstartTsToCreateStreamQueryImpl(SSelectStmt* pSelect, SHashObj* pUserAliasSet) { + SNode* pProj = nodesListGetNode(pSelect->pProjectionList, 0); if (NULL == pSelect->pWindow || (QUERY_NODE_FUNCTION == nodeType(pProj) && 0 == strcmp("_wstart", ((SFunctionNode*)pProj)->functionName))) { return TSDB_CODE_SUCCESS; @@ -5461,7 +5545,7 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pFunc->functionName, "_wstart"); - strcpy(pFunc->node.aliasName, pFunc->functionName); + getStreamQueryFirstProjectAliasName(pUserAliasSet, pFunc->node.aliasName, sizeof(pFunc->node.aliasName)); int32_t code = nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pFunc); @@ -5469,6 +5553,17 @@ static int32_t addWstartTsToCreateStreamQuery(SNode* pStmt) { return code; } +static int32_t addWstartTsToCreateStreamQuery(STranslateContext* pCxt, SNode* pStmt) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt; + SHashObj* pUserAliasSet = NULL; + int32_t code = checkProjectAlias(pCxt, pSelect->pProjectionList, &pUserAliasSet); + if (TSDB_CODE_SUCCESS == code) { + code = addWstartTsToCreateStreamQueryImpl(pSelect, pUserAliasSet); + } + taosHashCleanup(pUserAliasSet); + return code; +} + static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) { if (NULL == pStmt->pTags) { return TSDB_CODE_SUCCESS; @@ -5571,7 +5666,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; - int32_t code = addWstartTsToCreateStreamQuery(pStmt->pQuery); + int32_t code = addWstartTsToCreateStreamQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); } @@ -5673,7 +5768,7 @@ static int32_t readFromFile(char* pName, int32_t* len, char** buf) { if (s != *len) { taosCloseFile(&tfile); taosMemoryFreeClear(*buf); - return TSDB_CODE_TSC_APP_ERROR; + return TSDB_CODE_APP_ERROR; } taosCloseFile(&tfile); return TSDB_CODE_SUCCESS; @@ -6296,6 +6391,7 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { code = createShowCondition((SShowStmt*)pQuery->pRoot, pStmt); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; @@ -6349,6 +6445,7 @@ static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; pSelect->tagScan = true; nodesDestroyNode(pQuery->pRoot); @@ -6379,6 +6476,7 @@ static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery } } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pSelect; @@ -6398,6 +6496,7 @@ static int32_t rewriteShowVnodes(STranslateContext* pCxt, SQuery* pQuery) { } } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; @@ -6439,6 +6538,7 @@ static int32_t rewriteShowTableDist(STranslateContext* pCxt, SQuery* pQuery) { code = nodesListMakeStrictAppend(&pStmt->pProjectionList, createBlockDistFunc()); } if (TSDB_CODE_SUCCESS == code) { + pCxt->showRewrite = true; pQuery->showRewrite = true; nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = (SNode*)pStmt; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 7a56b0e0fa90b023c81f9180f5d02aecb879174c..fa091901b680041721936f7501fe23bb53b36305 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -377,7 +377,7 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, voi int32_t valLen = (int32_t)strlen(jsonValue); char* tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE); if (!tmp) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; + retCode = TSDB_CODE_OUT_OF_MEMORY; goto end; } val.type = TSDB_DATA_TYPE_NCHAR; @@ -474,7 +474,7 @@ static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicCondition return TSDB_CODE_SUCCESS; } -int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { +int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { if (NULL == pWhere) { return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 616793e8978ef3752e81e75c4bda412e11565707..b383d02006617433f66beeeb6788adbe2da2b485 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 456 +#define YYNOCODE 459 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy50; - int64_t yy93; - SNode* yy104; - bool yy185; - int32_t yy196; - EJoinType yy228; - EFillMode yy246; - SAlterOption yy557; - SNodeList* yy616; - SDataType yy640; - EOperatorType yy668; - int8_t yy695; - SToken yy737; - ENullOrder yy793; + EOperatorType yy20; + SNode* yy74; + ENullOrder yy109; + SToken yy317; + EOrder yy326; + bool yy335; + int8_t yy449; + int64_t yy531; + EJoinType yy630; + SAlterOption yy767; + EFillMode yy828; + int32_t yy856; + SNodeList* yy874; + SDataType yy898; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 707 -#define YYNRULE 538 -#define YYNTOKEN 321 -#define YY_MAX_SHIFT 706 -#define YY_MIN_SHIFTREDUCE 1049 -#define YY_MAX_SHIFTREDUCE 1586 -#define YY_ERROR_ACTION 1587 -#define YY_ACCEPT_ACTION 1588 -#define YY_NO_ACTION 1589 -#define YY_MIN_REDUCE 1590 -#define YY_MAX_REDUCE 2127 +#define YYNSTATE 714 +#define YYNRULE 539 +#define YYNTOKEN 324 +#define YY_MAX_SHIFT 713 +#define YY_MIN_SHIFTREDUCE 1055 +#define YY_MAX_SHIFTREDUCE 1593 +#define YY_ERROR_ACTION 1594 +#define YY_ACCEPT_ACTION 1595 +#define YY_NO_ACTION 1596 +#define YY_MIN_REDUCE 1597 +#define YY_MAX_REDUCE 2135 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,703 +216,790 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2544) +#define YY_ACTTAB_COUNT (2968) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 396, 2103, 349, 1858, 455, 2098, 456, 1626, 33, 274, - /* 10 */ 156, 578, 43, 41, 1517, 464, 168, 456, 1626, 1747, - /* 20 */ 357, 2102, 1368, 36, 35, 2099, 2101, 42, 40, 39, - /* 30 */ 38, 37, 595, 1447, 158, 1366, 1943, 329, 1844, 1698, - /* 40 */ 524, 540, 36, 35, 579, 2098, 42, 40, 39, 38, - /* 50 */ 37, 1394, 540, 522, 348, 520, 2098, 1855, 1442, 25, - /* 60 */ 2104, 174, 370, 16, 156, 2099, 561, 1961, 1791, 1792, - /* 70 */ 1374, 2104, 174, 1748, 2103, 575, 2099, 561, 2098, 572, - /* 80 */ 1912, 595, 612, 43, 41, 42, 40, 39, 38, 37, - /* 90 */ 159, 357, 1602, 1368, 2102, 1797, 12, 322, 2099, 2100, - /* 100 */ 11, 10, 350, 540, 1447, 1942, 1366, 2098, 62, 1977, - /* 110 */ 131, 1795, 100, 1944, 616, 1946, 1947, 611, 703, 606, - /* 120 */ 258, 603, 2104, 174, 171, 212, 2030, 2099, 561, 1442, - /* 130 */ 351, 2026, 1449, 1450, 16, 39, 38, 37, 473, 1476, - /* 140 */ 163, 1374, 2045, 402, 176, 1930, 490, 486, 482, 478, - /* 150 */ 209, 362, 2056, 80, 1790, 1792, 1926, 46, 58, 1797, - /* 160 */ 85, 1423, 1432, 1252, 1253, 1797, 326, 12, 2042, 255, - /* 170 */ 2038, 571, 361, 124, 570, 1795, 1741, 2098, 1393, 226, - /* 180 */ 1369, 1795, 1367, 317, 1922, 1928, 340, 81, 555, 703, - /* 190 */ 207, 1840, 559, 174, 1477, 606, 1490, 2099, 561, 1613, - /* 200 */ 58, 461, 182, 1449, 1450, 1372, 1373, 457, 1422, 1425, - /* 210 */ 1426, 1427, 1428, 1429, 1430, 1431, 608, 604, 1440, 1441, - /* 220 */ 1443, 1444, 1445, 1446, 1448, 1451, 2, 510, 509, 508, - /* 230 */ 560, 595, 1423, 1432, 2098, 128, 504, 1395, 82, 319, - /* 240 */ 503, 502, 585, 1912, 583, 1588, 501, 507, 1514, 559, - /* 250 */ 174, 1369, 500, 1367, 2099, 561, 206, 200, 1612, 205, - /* 260 */ 438, 550, 469, 1468, 1543, 32, 355, 1471, 1472, 1473, - /* 270 */ 1474, 1475, 1479, 1480, 1481, 1482, 1372, 1373, 198, 1422, - /* 280 */ 1425, 1426, 1427, 1428, 1429, 1430, 1431, 608, 604, 1440, - /* 290 */ 1441, 1443, 1444, 1445, 1446, 1448, 1451, 2, 170, 9, - /* 300 */ 43, 41, 1912, 506, 505, 177, 177, 46, 357, 372, - /* 310 */ 1368, 1784, 547, 1541, 1542, 1544, 1545, 190, 189, 1305, - /* 320 */ 1306, 1447, 1943, 1366, 1210, 638, 637, 636, 1214, 635, - /* 330 */ 1216, 1217, 634, 1219, 631, 1393, 1225, 628, 1227, 1228, - /* 340 */ 625, 622, 391, 97, 556, 551, 1442, 1101, 177, 1100, - /* 350 */ 540, 16, 454, 1961, 2098, 459, 1632, 132, 1374, 674, - /* 360 */ 672, 613, 211, 393, 389, 1737, 1912, 567, 612, 2104, - /* 370 */ 174, 43, 41, 1452, 2099, 561, 29, 1961, 1102, 357, - /* 380 */ 257, 1368, 36, 35, 12, 554, 42, 40, 39, 38, - /* 390 */ 37, 1942, 1447, 463, 1366, 1977, 459, 1632, 160, 1944, - /* 400 */ 616, 1946, 1947, 611, 596, 606, 703, 1943, 36, 35, - /* 410 */ 2045, 58, 42, 40, 39, 38, 37, 1442, 123, 264, - /* 420 */ 1449, 1450, 647, 50, 553, 494, 36, 35, 539, 1374, - /* 430 */ 42, 40, 39, 38, 37, 1745, 2041, 1170, 1961, 541, - /* 440 */ 2067, 147, 146, 644, 643, 642, 613, 91, 579, 1423, - /* 450 */ 1432, 1912, 58, 612, 560, 44, 2050, 1510, 2098, 36, - /* 460 */ 35, 1856, 1394, 42, 40, 39, 38, 37, 1369, 1738, - /* 470 */ 1367, 1396, 1172, 559, 174, 1611, 1942, 703, 2099, 561, - /* 480 */ 1977, 1345, 1346, 161, 1944, 616, 1946, 1947, 611, 1513, - /* 490 */ 606, 1449, 1450, 1372, 1373, 47, 1422, 1425, 1426, 1427, - /* 500 */ 1428, 1429, 1430, 1431, 608, 604, 1440, 1441, 1443, 1444, - /* 510 */ 1445, 1446, 1448, 1451, 2, 1553, 1082, 1610, 1609, 1912, - /* 520 */ 1423, 1432, 1591, 113, 1087, 1088, 112, 111, 110, 109, - /* 530 */ 108, 107, 106, 105, 104, 562, 2119, 265, 266, 1369, - /* 540 */ 9, 1367, 7, 113, 1722, 1608, 112, 111, 110, 109, - /* 550 */ 108, 107, 106, 105, 104, 1084, 1393, 1087, 1088, 177, - /* 560 */ 649, 1912, 1912, 155, 1372, 1373, 1478, 1422, 1425, 1426, - /* 570 */ 1427, 1428, 1429, 1430, 1431, 608, 604, 1440, 1441, 1443, - /* 580 */ 1444, 1445, 1446, 1448, 1451, 2, 43, 41, 568, 1912, - /* 590 */ 706, 515, 1392, 1101, 357, 1100, 1368, 1840, 319, 596, - /* 600 */ 177, 585, 596, 583, 281, 1521, 525, 1447, 184, 1366, - /* 610 */ 1943, 1393, 58, 52, 235, 641, 123, 596, 1607, 167, - /* 620 */ 225, 9, 572, 499, 1102, 696, 692, 688, 684, 279, - /* 630 */ 1745, 179, 1442, 1745, 360, 518, 80, 30, 168, 1374, - /* 640 */ 512, 1961, 156, 177, 1374, 224, 661, 1483, 1745, 613, - /* 650 */ 127, 1747, 647, 131, 1912, 1659, 612, 43, 41, 1740, - /* 660 */ 1845, 1723, 1912, 596, 1606, 357, 98, 1368, 1721, 272, - /* 670 */ 44, 147, 146, 644, 643, 642, 1734, 400, 1447, 1942, - /* 680 */ 1366, 1605, 64, 1977, 1730, 63, 100, 1944, 616, 1946, - /* 690 */ 1947, 611, 703, 606, 1745, 395, 134, 394, 143, 2001, - /* 700 */ 2030, 363, 592, 1442, 351, 2026, 1449, 1450, 1912, 156, - /* 710 */ 535, 574, 172, 2038, 2039, 1374, 129, 2043, 1747, 1732, - /* 720 */ 510, 509, 508, 473, 186, 1912, 1457, 572, 128, 504, - /* 730 */ 649, 1604, 1393, 503, 502, 1423, 1432, 1601, 260, 501, - /* 740 */ 507, 12, 1590, 36, 35, 500, 1396, 42, 40, 39, - /* 750 */ 38, 37, 1424, 1396, 1369, 1339, 1367, 229, 131, 227, - /* 760 */ 177, 77, 1775, 703, 76, 1736, 122, 121, 120, 119, - /* 770 */ 118, 117, 116, 115, 114, 1912, 1926, 1449, 1450, 1372, - /* 780 */ 1373, 1912, 1422, 1425, 1426, 1427, 1428, 1429, 1430, 1431, - /* 790 */ 608, 604, 1440, 1441, 1443, 1444, 1445, 1446, 1448, 1451, - /* 800 */ 2, 316, 498, 1391, 1922, 1928, 1423, 1432, 1600, 1840, - /* 810 */ 432, 2045, 598, 445, 2002, 606, 444, 173, 2038, 2039, - /* 820 */ 188, 129, 2043, 1797, 497, 1369, 1656, 1367, 1599, 1899, - /* 830 */ 1598, 416, 1533, 446, 31, 133, 418, 2040, 2001, 1796, - /* 840 */ 36, 35, 1597, 1596, 42, 40, 39, 38, 37, 1728, - /* 850 */ 1372, 1373, 1912, 1422, 1425, 1426, 1427, 1428, 1429, 1430, - /* 860 */ 1431, 608, 604, 1440, 1441, 1443, 1444, 1445, 1446, 1448, - /* 870 */ 1451, 2, 1912, 1424, 1912, 36, 35, 379, 330, 42, - /* 880 */ 40, 39, 38, 37, 1393, 645, 1912, 1912, 1788, 183, - /* 890 */ 406, 680, 679, 678, 677, 367, 234, 676, 675, 135, - /* 900 */ 670, 669, 668, 667, 666, 665, 664, 663, 149, 659, - /* 910 */ 658, 657, 366, 365, 654, 653, 652, 651, 650, 442, - /* 920 */ 1595, 1943, 437, 436, 435, 434, 431, 430, 429, 428, - /* 930 */ 427, 423, 422, 421, 420, 331, 413, 412, 411, 2102, - /* 940 */ 408, 407, 328, 157, 600, 596, 2002, 596, 292, 36, - /* 950 */ 35, 564, 1961, 42, 40, 39, 38, 37, 596, 401, - /* 960 */ 575, 410, 290, 66, 1912, 1912, 65, 612, 1854, 369, - /* 970 */ 312, 2103, 424, 145, 1943, 1594, 1745, 1853, 1745, 312, - /* 980 */ 1593, 596, 11, 10, 194, 451, 449, 1720, 1395, 1745, - /* 990 */ 1942, 572, 596, 646, 1977, 425, 1788, 100, 1944, 616, - /* 1000 */ 1946, 1947, 611, 6, 606, 1961, 139, 1827, 230, 171, - /* 1010 */ 540, 2030, 1745, 613, 2098, 351, 2026, 1699, 1912, 1912, - /* 1020 */ 612, 58, 131, 1745, 1912, 1424, 51, 596, 286, 2104, - /* 1030 */ 174, 1775, 596, 217, 2099, 561, 215, 2057, 1931, 1583, - /* 1040 */ 607, 471, 596, 1942, 233, 219, 472, 1977, 218, 1926, - /* 1050 */ 100, 1944, 616, 1946, 1947, 611, 364, 606, 1745, 99, - /* 1060 */ 596, 1943, 2118, 1745, 2030, 1576, 354, 353, 351, 2026, - /* 1070 */ 662, 640, 1715, 1745, 1742, 1377, 1382, 1922, 1928, 2064, - /* 1080 */ 596, 175, 2038, 2039, 83, 129, 2043, 1447, 606, 1375, - /* 1090 */ 1603, 1745, 1961, 403, 536, 647, 67, 74, 73, 399, - /* 1100 */ 613, 48, 181, 3, 1943, 1912, 404, 612, 137, 2070, - /* 1110 */ 125, 1745, 1442, 548, 147, 146, 644, 643, 642, 1376, - /* 1120 */ 315, 1930, 596, 387, 1374, 385, 381, 377, 374, 371, - /* 1130 */ 1942, 257, 1926, 1582, 1977, 1961, 576, 100, 1944, 616, - /* 1140 */ 1946, 1947, 611, 613, 606, 75, 1646, 96, 1912, 2118, - /* 1150 */ 612, 2030, 252, 1745, 1639, 351, 2026, 93, 419, 210, - /* 1160 */ 1922, 1928, 352, 528, 221, 596, 2092, 220, 511, 177, - /* 1170 */ 565, 606, 602, 1942, 596, 1943, 513, 1977, 60, 269, - /* 1180 */ 100, 1944, 616, 1946, 1947, 611, 596, 606, 591, 596, - /* 1190 */ 1637, 1933, 2118, 246, 2030, 1962, 1745, 563, 351, 2026, - /* 1200 */ 593, 596, 368, 594, 540, 1745, 1961, 1510, 2098, 2049, - /* 1210 */ 1585, 1586, 516, 223, 613, 275, 222, 1745, 1943, 1912, - /* 1220 */ 1745, 612, 1380, 2104, 174, 239, 45, 262, 2099, 561, - /* 1230 */ 140, 1540, 1745, 144, 1383, 145, 1378, 60, 1849, 1627, - /* 1240 */ 1935, 1785, 573, 2060, 1942, 254, 1130, 251, 1977, 1961, - /* 1250 */ 45, 100, 1944, 616, 1946, 1947, 611, 613, 606, 1386, - /* 1260 */ 1388, 1943, 1912, 2005, 612, 2030, 1379, 45, 620, 351, - /* 1270 */ 2026, 604, 1440, 1441, 1443, 1444, 1445, 1446, 241, 1316, - /* 1280 */ 267, 1131, 4, 588, 1633, 1943, 271, 1942, 1203, 655, - /* 1290 */ 1484, 1977, 1961, 144, 100, 1944, 616, 1946, 1947, 611, - /* 1300 */ 613, 606, 1, 1433, 145, 1912, 2003, 612, 2030, 373, - /* 1310 */ 378, 1150, 351, 2026, 656, 327, 1961, 126, 144, 1368, - /* 1320 */ 285, 1231, 282, 1332, 613, 187, 405, 1396, 1850, 1912, - /* 1330 */ 1942, 612, 1366, 698, 1977, 334, 1148, 100, 1944, 616, - /* 1340 */ 1946, 1947, 611, 1943, 606, 409, 1235, 440, 414, 599, - /* 1350 */ 1391, 2030, 426, 1842, 1942, 351, 2026, 1242, 1977, 433, - /* 1360 */ 447, 101, 1944, 616, 1946, 1947, 611, 1374, 606, 439, - /* 1370 */ 1240, 148, 441, 448, 1961, 2030, 191, 450, 452, 2029, - /* 1380 */ 2026, 1397, 613, 453, 462, 1399, 1943, 1912, 465, 612, - /* 1390 */ 197, 199, 466, 1398, 467, 1400, 468, 335, 202, 333, - /* 1400 */ 332, 470, 496, 204, 474, 78, 498, 79, 208, 1104, - /* 1410 */ 491, 492, 1942, 493, 1943, 703, 1977, 1961, 495, 101, - /* 1420 */ 1944, 616, 1946, 1947, 611, 613, 606, 102, 497, 1735, - /* 1430 */ 1912, 318, 612, 2030, 214, 1731, 216, 601, 2026, 527, - /* 1440 */ 150, 151, 529, 1733, 1729, 1961, 152, 153, 228, 530, - /* 1450 */ 1889, 231, 537, 610, 549, 614, 582, 5, 1912, 1977, - /* 1460 */ 612, 2061, 101, 1944, 616, 1946, 1947, 611, 283, 606, - /* 1470 */ 534, 2071, 544, 1943, 2076, 546, 2030, 1369, 237, 1367, - /* 1480 */ 321, 2026, 240, 1942, 531, 2075, 341, 1977, 558, 552, - /* 1490 */ 307, 1944, 616, 1946, 1947, 611, 609, 606, 597, 1995, - /* 1500 */ 545, 543, 1372, 1373, 1961, 542, 250, 569, 342, 566, - /* 1510 */ 2121, 245, 613, 1510, 130, 1395, 2046, 1912, 345, 612, - /* 1520 */ 577, 141, 142, 580, 581, 1888, 1860, 586, 259, 347, - /* 1530 */ 88, 1943, 589, 248, 164, 2052, 249, 247, 1746, 590, - /* 1540 */ 90, 57, 1942, 2011, 92, 1789, 1977, 284, 1716, 101, - /* 1550 */ 1944, 616, 1946, 1947, 611, 2097, 606, 253, 287, 1943, - /* 1560 */ 618, 278, 1961, 2030, 699, 700, 49, 301, 2027, 310, - /* 1570 */ 613, 309, 702, 289, 291, 1912, 1906, 612, 1905, 311, - /* 1580 */ 71, 1904, 1903, 72, 1900, 1943, 375, 376, 1360, 1361, - /* 1590 */ 1961, 180, 1898, 380, 382, 383, 384, 1897, 613, 386, - /* 1600 */ 1942, 1896, 388, 1912, 1977, 612, 1895, 160, 1944, 616, - /* 1610 */ 1946, 1947, 611, 1894, 606, 390, 1961, 1335, 392, 1334, - /* 1620 */ 1871, 1870, 397, 1869, 613, 398, 1868, 1296, 1942, 1912, - /* 1630 */ 1835, 612, 1977, 1834, 1832, 300, 1944, 616, 1946, 1947, - /* 1640 */ 611, 136, 606, 1831, 1830, 1833, 1829, 1828, 1826, 2068, - /* 1650 */ 1825, 1824, 185, 415, 1942, 1943, 1823, 417, 1977, 1822, - /* 1660 */ 1821, 161, 1944, 616, 1946, 1947, 611, 1820, 606, 1819, - /* 1670 */ 1818, 1817, 1816, 1815, 1943, 1298, 1802, 138, 1807, 557, - /* 1680 */ 1814, 1813, 1812, 1811, 1810, 1809, 1961, 1808, 1806, 1805, - /* 1690 */ 1804, 346, 443, 1799, 613, 1803, 1801, 1800, 1798, 1912, - /* 1700 */ 1178, 612, 1661, 1660, 1658, 1961, 1622, 192, 195, 1932, - /* 1710 */ 193, 458, 169, 610, 2120, 1090, 69, 1621, 1912, 1089, - /* 1720 */ 612, 460, 1884, 1878, 1942, 1867, 196, 1866, 1977, 203, - /* 1730 */ 70, 308, 1944, 616, 1946, 1947, 611, 1943, 606, 201, - /* 1740 */ 1852, 1724, 1657, 1942, 1655, 475, 477, 1977, 1653, 479, - /* 1750 */ 307, 1944, 616, 1946, 1947, 611, 476, 606, 1943, 1996, - /* 1760 */ 480, 1651, 1649, 1123, 483, 481, 484, 485, 1961, 487, - /* 1770 */ 489, 1636, 488, 356, 1635, 1618, 613, 1726, 1246, 1245, - /* 1780 */ 1943, 1912, 1725, 612, 59, 1169, 1168, 1167, 1166, 1961, - /* 1790 */ 1165, 671, 1160, 673, 358, 1162, 1647, 613, 1161, 1159, - /* 1800 */ 336, 1943, 1912, 1640, 612, 337, 1942, 1638, 514, 338, - /* 1810 */ 1977, 1961, 213, 308, 1944, 616, 1946, 1947, 611, 613, - /* 1820 */ 606, 517, 1617, 1943, 1912, 519, 612, 1942, 1616, 521, - /* 1830 */ 1615, 1977, 1961, 523, 308, 1944, 616, 1946, 1947, 611, - /* 1840 */ 613, 606, 1352, 103, 1943, 1912, 1883, 612, 24, 526, - /* 1850 */ 1341, 1877, 1865, 1977, 1961, 532, 303, 1944, 616, 1946, - /* 1860 */ 1947, 611, 613, 606, 1863, 17, 2103, 1912, 26, 612, - /* 1870 */ 1942, 56, 14, 1555, 1977, 1961, 53, 293, 1944, 616, - /* 1880 */ 1946, 1947, 611, 613, 606, 232, 238, 1943, 1912, 236, - /* 1890 */ 612, 162, 1942, 533, 154, 339, 1977, 1539, 1532, 294, - /* 1900 */ 1944, 616, 1946, 1947, 611, 538, 606, 244, 242, 27, - /* 1910 */ 243, 28, 1933, 1942, 84, 1943, 61, 1977, 1961, 19, - /* 1920 */ 295, 1944, 616, 1946, 1947, 611, 613, 606, 1575, 1576, - /* 1930 */ 1570, 1912, 1569, 612, 343, 1574, 1573, 344, 1507, 1506, - /* 1940 */ 256, 1943, 18, 165, 1864, 1862, 1961, 55, 1861, 20, - /* 1950 */ 1350, 261, 1537, 263, 613, 1349, 1942, 54, 584, 1912, - /* 1960 */ 1977, 612, 1859, 299, 1944, 616, 1946, 1947, 611, 15, - /* 1970 */ 606, 587, 1961, 268, 86, 1851, 87, 89, 270, 93, - /* 1980 */ 613, 273, 21, 1459, 1942, 1912, 10, 612, 1977, 1384, - /* 1990 */ 1469, 304, 1944, 616, 1946, 1947, 611, 8, 606, 1943, - /* 2000 */ 1458, 1437, 1980, 605, 166, 1435, 34, 178, 1434, 1407, - /* 2010 */ 1942, 615, 13, 22, 1977, 1415, 1943, 296, 1944, 616, - /* 2020 */ 1946, 1947, 611, 1232, 606, 23, 617, 619, 621, 359, - /* 2030 */ 1961, 623, 1229, 1226, 626, 624, 627, 629, 613, 1220, - /* 2040 */ 1218, 632, 630, 1912, 1224, 612, 633, 1961, 1223, 1209, - /* 2050 */ 1222, 1221, 94, 1241, 95, 613, 276, 639, 1237, 1943, - /* 2060 */ 1912, 68, 612, 1156, 1121, 1155, 648, 1154, 1942, 1153, - /* 2070 */ 1152, 1176, 1977, 1151, 1149, 305, 1944, 616, 1946, 1947, - /* 2080 */ 611, 1147, 606, 1943, 1146, 1942, 1145, 277, 660, 1977, - /* 2090 */ 1961, 1143, 297, 1944, 616, 1946, 1947, 611, 613, 606, - /* 2100 */ 1142, 1141, 1140, 1912, 1139, 612, 1138, 1137, 1127, 1136, - /* 2110 */ 1173, 1171, 1133, 1132, 1961, 1129, 1128, 1654, 1126, 681, - /* 2120 */ 1652, 1650, 613, 682, 683, 685, 1943, 1912, 1942, 612, - /* 2130 */ 687, 691, 1977, 689, 1648, 306, 1944, 616, 1946, 1947, - /* 2140 */ 611, 693, 606, 1943, 686, 690, 695, 694, 1634, 697, - /* 2150 */ 1079, 1614, 1942, 280, 701, 705, 1977, 1961, 704, 298, - /* 2160 */ 1944, 616, 1946, 1947, 611, 613, 606, 1370, 288, 1943, - /* 2170 */ 1912, 1589, 612, 1589, 1961, 1589, 1589, 1589, 1589, 1589, - /* 2180 */ 1589, 1589, 613, 1589, 1589, 1589, 1943, 1912, 1589, 612, - /* 2190 */ 1589, 1589, 1589, 1589, 1589, 1942, 1589, 1589, 1589, 1977, - /* 2200 */ 1961, 1589, 313, 1944, 616, 1946, 1947, 611, 613, 606, - /* 2210 */ 1589, 1589, 1942, 1912, 1589, 612, 1977, 1961, 1589, 314, - /* 2220 */ 1944, 616, 1946, 1947, 611, 613, 606, 1589, 1589, 1943, - /* 2230 */ 1912, 1589, 612, 1589, 1589, 1589, 1589, 1589, 1942, 1589, - /* 2240 */ 1589, 1589, 1977, 1589, 1589, 1955, 1944, 616, 1946, 1947, - /* 2250 */ 611, 1943, 606, 1589, 1589, 1942, 1589, 1589, 1589, 1977, - /* 2260 */ 1961, 1589, 1954, 1944, 616, 1946, 1947, 611, 613, 606, - /* 2270 */ 1589, 1589, 1589, 1912, 1589, 612, 1589, 1589, 1589, 1589, - /* 2280 */ 1589, 1589, 1961, 1589, 1589, 1589, 1589, 1589, 1589, 1589, - /* 2290 */ 613, 1589, 1589, 1589, 1589, 1912, 1589, 612, 1942, 1589, - /* 2300 */ 1589, 1589, 1977, 1589, 1589, 1953, 1944, 616, 1946, 1947, - /* 2310 */ 611, 1589, 606, 1943, 1589, 1589, 1589, 1589, 1589, 1589, - /* 2320 */ 1942, 1589, 1589, 1589, 1977, 1589, 1589, 323, 1944, 616, - /* 2330 */ 1946, 1947, 611, 1589, 606, 1943, 1589, 1589, 1589, 1589, - /* 2340 */ 1589, 1589, 1589, 1589, 1961, 1589, 1589, 1589, 1589, 1589, - /* 2350 */ 1589, 1589, 613, 1589, 1589, 1589, 1589, 1912, 1589, 612, - /* 2360 */ 1589, 1589, 1589, 1589, 1589, 1589, 1961, 1589, 1589, 1589, - /* 2370 */ 1589, 1589, 1589, 1589, 613, 1589, 1589, 1589, 1589, 1912, - /* 2380 */ 1589, 612, 1942, 1589, 1589, 1589, 1977, 1589, 1589, 324, - /* 2390 */ 1944, 616, 1946, 1947, 611, 1589, 606, 1943, 1589, 1589, - /* 2400 */ 1589, 1589, 1589, 1589, 1942, 1589, 1589, 1589, 1977, 1589, - /* 2410 */ 1589, 320, 1944, 616, 1946, 1947, 611, 1943, 606, 1589, - /* 2420 */ 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1961, 1589, - /* 2430 */ 1589, 1589, 1589, 1589, 1589, 1589, 613, 1589, 1589, 1589, - /* 2440 */ 1589, 1912, 1589, 612, 1589, 1589, 1589, 1589, 1961, 1589, - /* 2450 */ 1589, 1589, 1589, 1589, 1589, 1589, 613, 1589, 1589, 1589, - /* 2460 */ 1943, 1912, 1589, 612, 1589, 1589, 1942, 1589, 1589, 1589, - /* 2470 */ 1977, 1589, 1589, 325, 1944, 616, 1946, 1947, 611, 1589, - /* 2480 */ 606, 1589, 1589, 1589, 1589, 1589, 614, 1589, 1589, 1589, - /* 2490 */ 1977, 1961, 1589, 303, 1944, 616, 1946, 1947, 611, 613, - /* 2500 */ 606, 1589, 1589, 1589, 1912, 1589, 612, 1589, 1589, 1589, - /* 2510 */ 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, - /* 2520 */ 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1589, 1942, - /* 2530 */ 1589, 1589, 1589, 1977, 1589, 1589, 302, 1944, 616, 1946, - /* 2540 */ 1947, 611, 1589, 606, + /* 0 */ 460, 354, 461, 1633, 1739, 469, 1595, 461, 1633, 157, + /* 10 */ 542, 82, 45, 43, 1523, 1937, 169, 159, 1752, 1609, + /* 20 */ 362, 1401, 1374, 38, 37, 129, 1933, 44, 42, 41, + /* 30 */ 40, 39, 602, 1453, 1745, 1372, 1666, 334, 1850, 602, + /* 40 */ 1937, 367, 38, 37, 1796, 1798, 44, 42, 41, 40, + /* 50 */ 39, 1933, 407, 1846, 1929, 1935, 345, 583, 1448, 27, + /* 60 */ 1527, 31, 1728, 18, 183, 613, 1399, 38, 37, 377, + /* 70 */ 1380, 44, 42, 41, 40, 39, 171, 1950, 1803, 1929, + /* 80 */ 1935, 357, 64, 45, 43, 355, 1399, 1741, 133, 1790, + /* 90 */ 613, 362, 320, 1374, 1801, 14, 459, 327, 1933, 464, + /* 100 */ 1639, 514, 513, 512, 1453, 365, 1372, 48, 1968, 130, + /* 110 */ 508, 547, 466, 157, 507, 2106, 586, 710, 462, 506, + /* 120 */ 511, 1919, 1752, 619, 478, 505, 1929, 1935, 566, 1448, + /* 130 */ 2112, 175, 1455, 1456, 18, 2107, 572, 613, 1482, 468, + /* 140 */ 160, 1380, 464, 1639, 583, 1704, 1949, 174, 2045, 2046, + /* 150 */ 1984, 131, 2050, 102, 1951, 623, 1953, 1954, 618, 571, + /* 160 */ 613, 1429, 1438, 2106, 259, 172, 14, 2037, 514, 513, + /* 170 */ 512, 356, 2033, 60, 478, 133, 130, 508, 570, 175, + /* 180 */ 1375, 507, 1373, 2107, 572, 177, 506, 511, 710, 605, + /* 190 */ 1938, 2009, 505, 2063, 1483, 227, 44, 42, 41, 40, + /* 200 */ 39, 1933, 1088, 1455, 1456, 1378, 1379, 1430, 1428, 1431, + /* 210 */ 1432, 1433, 1434, 1435, 1436, 1437, 615, 611, 1446, 1447, + /* 220 */ 1449, 1450, 1451, 1452, 1454, 1457, 2, 1310, 1311, 1929, + /* 230 */ 1935, 1400, 1429, 1438, 257, 2045, 582, 368, 126, 581, + /* 240 */ 613, 1090, 2106, 1093, 1094, 157, 60, 1107, 1803, 1106, + /* 250 */ 339, 1375, 1550, 1373, 1752, 84, 322, 570, 175, 532, + /* 260 */ 443, 530, 2107, 572, 1802, 34, 360, 1477, 1478, 1479, + /* 270 */ 1480, 1481, 1485, 1486, 1487, 1488, 1378, 1379, 1108, 1428, + /* 280 */ 1431, 1432, 1433, 1434, 1435, 1436, 1437, 615, 611, 1446, + /* 290 */ 1447, 1449, 1450, 1451, 1452, 1454, 1457, 2, 2111, 11, + /* 300 */ 45, 43, 558, 1548, 1549, 1551, 1552, 11, 362, 9, + /* 310 */ 1374, 340, 602, 338, 337, 1401, 501, 583, 191, 190, + /* 320 */ 503, 1453, 178, 1372, 1214, 645, 644, 643, 1218, 642, + /* 330 */ 1220, 1221, 641, 1223, 638, 1598, 1229, 635, 1231, 1232, + /* 340 */ 632, 629, 502, 1257, 1258, 260, 1448, 235, 133, 187, + /* 350 */ 2111, 18, 228, 610, 2106, 99, 115, 11, 1380, 114, + /* 360 */ 113, 112, 111, 110, 109, 108, 107, 106, 169, 134, + /* 370 */ 2110, 45, 43, 1458, 2107, 2109, 607, 1742, 2009, 362, + /* 380 */ 401, 1374, 60, 14, 87, 79, 85, 48, 78, 60, + /* 390 */ 1851, 1583, 1453, 115, 1372, 178, 114, 113, 112, 111, + /* 400 */ 110, 109, 108, 107, 106, 710, 585, 173, 2045, 2046, + /* 410 */ 49, 131, 2050, 213, 590, 1351, 1352, 1448, 590, 2111, + /* 420 */ 1455, 1456, 547, 2106, 353, 82, 2106, 1861, 164, 1380, + /* 430 */ 1107, 1862, 1106, 561, 495, 491, 487, 483, 210, 2110, + /* 440 */ 1374, 2112, 175, 2107, 2108, 212, 2107, 572, 1746, 1429, + /* 450 */ 1438, 267, 268, 1372, 46, 1597, 266, 1399, 259, 38, + /* 460 */ 37, 1108, 1803, 44, 42, 41, 40, 39, 1375, 331, + /* 470 */ 1373, 1402, 1402, 400, 83, 399, 710, 208, 1801, 124, + /* 480 */ 123, 122, 121, 120, 119, 118, 117, 116, 1380, 2052, + /* 490 */ 1620, 1455, 1456, 1378, 1379, 1474, 1428, 1431, 1432, 1433, + /* 500 */ 1434, 1435, 1436, 1437, 615, 611, 1446, 1447, 1449, 1450, + /* 510 */ 1451, 1452, 1454, 1457, 2, 2049, 567, 562, 556, 1463, + /* 520 */ 1429, 1438, 38, 37, 1484, 1399, 44, 42, 41, 40, + /* 530 */ 39, 178, 60, 571, 1919, 710, 178, 2106, 178, 1375, + /* 540 */ 396, 1373, 1400, 1968, 207, 201, 1726, 206, 35, 276, + /* 550 */ 474, 565, 570, 175, 157, 589, 1399, 2107, 572, 178, + /* 560 */ 1950, 398, 394, 1753, 1378, 1379, 199, 1428, 1431, 1432, + /* 570 */ 1433, 1434, 1435, 1436, 1437, 615, 611, 1446, 1447, 1449, + /* 580 */ 1450, 1451, 1452, 1454, 1457, 2, 45, 43, 603, 603, + /* 590 */ 564, 1968, 1797, 1798, 362, 32, 1374, 547, 1375, 620, + /* 600 */ 1373, 2106, 54, 180, 1919, 1489, 619, 1453, 657, 1372, + /* 610 */ 1560, 41, 40, 39, 237, 184, 2112, 175, 1803, 1750, + /* 620 */ 1750, 2107, 572, 1378, 1379, 366, 375, 510, 509, 1949, + /* 630 */ 2110, 603, 1448, 1984, 1801, 8, 102, 1951, 623, 1953, + /* 640 */ 1954, 618, 1727, 613, 1380, 125, 136, 2052, 143, 2008, + /* 650 */ 2037, 1950, 499, 2052, 356, 2033, 374, 45, 43, 1402, + /* 660 */ 519, 528, 1750, 603, 1619, 362, 1430, 1374, 547, 46, + /* 670 */ 13, 12, 2106, 2048, 526, 529, 524, 125, 1453, 2047, + /* 680 */ 1372, 178, 1968, 93, 504, 1846, 1174, 2112, 175, 226, + /* 690 */ 586, 710, 2107, 572, 1750, 1919, 185, 619, 547, 681, + /* 700 */ 679, 1398, 2106, 1448, 522, 1743, 1455, 1456, 1919, 516, + /* 710 */ 135, 603, 648, 2008, 225, 1380, 1618, 2112, 175, 1617, + /* 720 */ 1949, 1176, 2107, 572, 1984, 405, 1846, 102, 1951, 623, + /* 730 */ 1953, 1954, 618, 603, 613, 1429, 1438, 189, 657, 172, + /* 740 */ 14, 2037, 1750, 603, 603, 356, 2033, 406, 229, 655, + /* 750 */ 66, 1780, 649, 65, 1375, 1794, 1373, 415, 429, 503, + /* 760 */ 1919, 1616, 710, 1919, 1750, 1496, 1640, 2064, 148, 147, + /* 770 */ 652, 651, 650, 145, 1750, 1750, 1380, 1455, 1456, 1378, + /* 780 */ 1379, 502, 1428, 1431, 1432, 1433, 1434, 1435, 1436, 1437, + /* 790 */ 615, 611, 1446, 1447, 1449, 1450, 1451, 1452, 1454, 1457, + /* 800 */ 2, 319, 236, 1397, 603, 1919, 1429, 1438, 1615, 653, + /* 810 */ 437, 654, 1794, 450, 1794, 705, 449, 290, 430, 669, + /* 820 */ 1780, 670, 50, 1720, 3, 1375, 1663, 1373, 1614, 1613, + /* 830 */ 1612, 421, 146, 451, 33, 1750, 423, 1093, 1094, 1516, + /* 840 */ 38, 37, 1725, 1833, 44, 42, 41, 40, 39, 1399, + /* 850 */ 1378, 1379, 1919, 1428, 1431, 1432, 1433, 1434, 1435, 1436, + /* 860 */ 1437, 615, 611, 1446, 1447, 1449, 1450, 1451, 1452, 1454, + /* 870 */ 1457, 2, 1919, 1919, 1919, 38, 37, 335, 1735, 44, + /* 880 */ 42, 41, 40, 39, 53, 218, 2057, 1516, 216, 411, + /* 890 */ 574, 687, 686, 685, 684, 372, 1520, 683, 682, 137, + /* 900 */ 677, 676, 675, 674, 673, 672, 671, 150, 667, 666, + /* 910 */ 665, 371, 370, 662, 661, 660, 659, 658, 139, 447, + /* 920 */ 127, 1383, 442, 441, 440, 439, 436, 435, 434, 433, + /* 930 */ 432, 428, 427, 426, 425, 336, 418, 417, 416, 69, + /* 940 */ 413, 412, 333, 158, 578, 535, 38, 37, 296, 655, + /* 950 */ 44, 42, 41, 40, 39, 220, 222, 1950, 219, 221, + /* 960 */ 1611, 1608, 294, 68, 224, 1610, 67, 223, 148, 147, + /* 970 */ 652, 651, 650, 145, 1737, 38, 37, 1950, 603, 44, + /* 980 */ 42, 41, 40, 39, 195, 456, 454, 547, 1968, 77, + /* 990 */ 1430, 2106, 476, 583, 424, 575, 620, 1607, 1606, 1605, + /* 1000 */ 98, 1919, 408, 619, 1919, 1919, 2112, 175, 1968, 1750, + /* 1010 */ 95, 2107, 572, 603, 52, 409, 620, 603, 603, 546, + /* 1020 */ 60, 1919, 1604, 619, 133, 1906, 1949, 477, 62, 241, + /* 1030 */ 1984, 1747, 141, 102, 1951, 623, 1953, 1954, 618, 1590, + /* 1040 */ 613, 1919, 1919, 1919, 1750, 2126, 1949, 2037, 1750, 1750, + /* 1050 */ 1984, 356, 2033, 102, 1951, 623, 1953, 1954, 618, 101, + /* 1060 */ 613, 1382, 2071, 1539, 603, 2126, 1919, 2037, 1386, 1950, + /* 1070 */ 655, 356, 2033, 384, 1603, 1592, 1593, 1602, 543, 1733, + /* 1080 */ 1547, 243, 2084, 176, 2045, 2046, 232, 131, 2050, 148, + /* 1090 */ 147, 652, 651, 650, 145, 1750, 614, 76, 75, 404, + /* 1100 */ 1968, 156, 182, 603, 647, 603, 1705, 2077, 620, 559, + /* 1110 */ 603, 1653, 1950, 1919, 47, 619, 1601, 587, 1919, 271, + /* 1120 */ 318, 1919, 254, 392, 598, 390, 386, 382, 379, 376, + /* 1130 */ 603, 13, 12, 515, 1750, 1589, 1750, 1519, 1949, 603, + /* 1140 */ 603, 1750, 1984, 1968, 600, 102, 1951, 623, 1953, 1954, + /* 1150 */ 618, 620, 613, 601, 277, 1950, 1919, 2126, 619, 2037, + /* 1160 */ 1919, 1750, 1646, 356, 2033, 1644, 1321, 579, 1600, 178, + /* 1170 */ 1750, 1750, 38, 37, 554, 264, 44, 42, 41, 40, + /* 1180 */ 39, 1949, 603, 211, 517, 1984, 1968, 520, 102, 1951, + /* 1190 */ 623, 1953, 1954, 618, 620, 613, 369, 248, 1940, 1919, + /* 1200 */ 2126, 619, 2037, 142, 144, 146, 356, 2033, 1385, 373, + /* 1210 */ 62, 47, 1919, 1750, 1634, 47, 576, 2100, 627, 359, + /* 1220 */ 358, 144, 1950, 146, 1949, 663, 128, 269, 1984, 1388, + /* 1230 */ 144, 102, 1951, 623, 1953, 1954, 618, 664, 613, 1136, + /* 1240 */ 1453, 1969, 1381, 2126, 1855, 2037, 1942, 1155, 1791, 356, + /* 1250 */ 2033, 2067, 584, 1968, 253, 595, 273, 1207, 256, 1153, + /* 1260 */ 2056, 620, 1490, 1439, 1, 1448, 1919, 289, 619, 4, + /* 1270 */ 1235, 378, 332, 1239, 1137, 1246, 1950, 1380, 1244, 383, + /* 1280 */ 1338, 188, 149, 284, 410, 1402, 414, 445, 1856, 419, + /* 1290 */ 1397, 1949, 431, 1848, 438, 1984, 444, 446, 102, 1951, + /* 1300 */ 623, 1953, 1954, 618, 452, 613, 192, 1968, 453, 455, + /* 1310 */ 2012, 457, 2037, 1403, 1405, 620, 356, 2033, 458, 467, + /* 1320 */ 1919, 470, 619, 198, 609, 471, 1404, 200, 472, 1406, + /* 1330 */ 473, 475, 203, 479, 205, 80, 1110, 81, 209, 496, + /* 1340 */ 497, 498, 321, 105, 1896, 1949, 500, 534, 1895, 1984, + /* 1350 */ 536, 538, 102, 1951, 623, 1953, 1954, 618, 1740, 613, + /* 1360 */ 230, 215, 1950, 1736, 2010, 217, 2037, 151, 152, 1738, + /* 1370 */ 356, 2033, 1734, 153, 154, 537, 285, 541, 233, 544, + /* 1380 */ 2068, 7, 560, 2083, 2082, 593, 569, 1389, 2059, 1384, + /* 1390 */ 551, 557, 249, 1968, 247, 346, 552, 550, 2078, 563, + /* 1400 */ 165, 620, 239, 250, 549, 251, 1919, 347, 619, 580, + /* 1410 */ 577, 242, 1392, 1394, 2129, 1516, 2105, 132, 1401, 255, + /* 1420 */ 252, 588, 261, 350, 611, 1446, 1447, 1449, 1450, 1451, + /* 1430 */ 1452, 1949, 2053, 286, 591, 1984, 1950, 592, 102, 1951, + /* 1440 */ 623, 1953, 1954, 618, 1867, 613, 1866, 1865, 287, 352, + /* 1450 */ 606, 596, 2037, 597, 90, 288, 356, 2033, 92, 1751, + /* 1460 */ 59, 2018, 94, 1795, 1721, 625, 706, 1968, 291, 707, + /* 1470 */ 709, 51, 300, 315, 280, 620, 323, 314, 304, 1950, + /* 1480 */ 1919, 324, 619, 295, 293, 1913, 1912, 73, 1911, 1910, + /* 1490 */ 74, 1907, 380, 1366, 381, 1367, 181, 385, 1905, 387, + /* 1500 */ 388, 389, 1904, 391, 1903, 1949, 393, 1902, 1901, 1984, + /* 1510 */ 1968, 397, 103, 1951, 623, 1953, 1954, 618, 620, 613, + /* 1520 */ 1341, 395, 1950, 1919, 1340, 619, 2037, 402, 403, 1876, + /* 1530 */ 2036, 2033, 1878, 1877, 1875, 1841, 1840, 1301, 1838, 138, + /* 1540 */ 1837, 1836, 1839, 1835, 1834, 1832, 1831, 1830, 1949, 420, + /* 1550 */ 1950, 186, 1984, 1968, 1829, 103, 1951, 623, 1953, 1954, + /* 1560 */ 618, 620, 613, 422, 1828, 1827, 1919, 1826, 619, 2037, + /* 1570 */ 140, 1813, 1812, 608, 2033, 1825, 1824, 1823, 1822, 1821, + /* 1580 */ 1820, 1968, 1819, 1818, 1817, 1816, 1815, 1814, 1811, 617, + /* 1590 */ 1810, 621, 1809, 1808, 1919, 1984, 619, 1807, 103, 1951, + /* 1600 */ 623, 1953, 1954, 618, 1806, 613, 1805, 448, 1804, 1182, + /* 1610 */ 1668, 1303, 2037, 193, 1667, 194, 326, 2033, 1665, 1949, + /* 1620 */ 1629, 1096, 713, 1984, 1628, 170, 312, 1951, 623, 1953, + /* 1630 */ 1954, 618, 616, 613, 604, 2002, 283, 196, 1950, 71, + /* 1640 */ 1095, 1939, 197, 463, 72, 465, 1891, 1885, 1874, 204, + /* 1650 */ 202, 168, 1873, 1858, 1729, 1129, 1664, 703, 699, 695, + /* 1660 */ 691, 281, 1662, 480, 481, 1660, 484, 485, 482, 1968, + /* 1670 */ 1658, 486, 488, 489, 490, 1656, 492, 620, 493, 494, + /* 1680 */ 1643, 1950, 1919, 1642, 619, 1625, 1731, 1730, 1250, 678, + /* 1690 */ 1251, 1165, 1173, 1172, 1171, 1170, 1654, 100, 1950, 1167, + /* 1700 */ 274, 341, 1166, 61, 1647, 680, 1164, 1949, 1645, 521, + /* 1710 */ 342, 1984, 1968, 518, 161, 1951, 623, 1953, 1954, 618, + /* 1720 */ 620, 613, 343, 214, 1624, 1919, 523, 619, 1623, 1968, + /* 1730 */ 525, 1622, 527, 599, 104, 1358, 1356, 620, 1355, 26, + /* 1740 */ 531, 1890, 1919, 1884, 619, 155, 1347, 539, 1872, 1870, + /* 1750 */ 1949, 19, 1950, 2111, 1984, 548, 2074, 162, 1951, 623, + /* 1760 */ 1953, 1954, 618, 16, 613, 555, 234, 1949, 1950, 1562, + /* 1770 */ 262, 1984, 55, 240, 103, 1951, 623, 1953, 1954, 618, + /* 1780 */ 540, 613, 545, 1968, 28, 344, 553, 1345, 2037, 231, + /* 1790 */ 58, 620, 238, 2034, 1546, 245, 1919, 163, 619, 1968, + /* 1800 */ 244, 5, 6, 246, 29, 1940, 30, 620, 1538, 573, + /* 1810 */ 2127, 86, 1919, 1582, 619, 1583, 20, 63, 21, 1513, + /* 1820 */ 1577, 1949, 1576, 348, 1581, 1984, 1580, 349, 161, 1951, + /* 1830 */ 623, 1953, 1954, 618, 1512, 613, 1871, 1949, 1950, 57, + /* 1840 */ 258, 1984, 1869, 166, 306, 1951, 623, 1953, 1954, 618, + /* 1850 */ 1868, 613, 22, 1857, 1950, 263, 1544, 56, 265, 270, + /* 1860 */ 17, 88, 89, 91, 275, 23, 95, 10, 12, 1968, + /* 1870 */ 2075, 1390, 594, 1475, 1987, 612, 1465, 620, 1464, 272, + /* 1880 */ 167, 36, 1919, 1421, 619, 1968, 1443, 179, 568, 15, + /* 1890 */ 351, 622, 24, 620, 1441, 1440, 1413, 1950, 1919, 25, + /* 1900 */ 619, 1236, 626, 364, 628, 1213, 1233, 1949, 624, 630, + /* 1910 */ 631, 1984, 1230, 633, 162, 1951, 623, 1953, 1954, 618, + /* 1920 */ 634, 613, 1224, 1949, 636, 637, 639, 1984, 1968, 640, + /* 1930 */ 313, 1951, 623, 1953, 1954, 618, 617, 613, 1228, 1222, + /* 1940 */ 96, 1919, 646, 619, 278, 1245, 1227, 1226, 1225, 97, + /* 1950 */ 70, 1241, 1127, 656, 1950, 1161, 1160, 1159, 1158, 1157, + /* 1960 */ 1156, 1154, 1152, 1151, 668, 1150, 1949, 2128, 1180, 1148, + /* 1970 */ 1984, 1147, 1145, 312, 1951, 623, 1953, 1954, 618, 279, + /* 1980 */ 613, 1146, 2003, 1144, 1143, 1968, 1142, 1177, 1175, 1139, + /* 1990 */ 361, 1138, 1135, 620, 1134, 1133, 1132, 1661, 1919, 688, + /* 2000 */ 619, 690, 1659, 692, 694, 1950, 689, 1657, 693, 696, + /* 2010 */ 698, 697, 1655, 700, 701, 702, 1641, 704, 1085, 1621, + /* 2020 */ 282, 708, 711, 1949, 712, 1376, 1950, 1984, 292, 1596, + /* 2030 */ 313, 1951, 623, 1953, 1954, 618, 1968, 613, 1596, 1596, + /* 2040 */ 1596, 363, 1596, 1596, 620, 1596, 1596, 1596, 1950, 1919, + /* 2050 */ 1596, 619, 1596, 1596, 1596, 1596, 1596, 1968, 1596, 1596, + /* 2060 */ 1596, 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, 1596, + /* 2070 */ 1919, 1596, 619, 1596, 1949, 1596, 1596, 1596, 1984, 1968, + /* 2080 */ 1596, 313, 1951, 623, 1953, 1954, 618, 620, 613, 1596, + /* 2090 */ 1596, 1596, 1919, 1596, 619, 533, 1596, 1596, 1596, 1984, + /* 2100 */ 1596, 1596, 308, 1951, 623, 1953, 1954, 618, 1596, 613, + /* 2110 */ 1596, 1950, 1596, 1596, 1596, 1596, 1596, 1949, 1596, 1596, + /* 2120 */ 1596, 1984, 1596, 1596, 297, 1951, 623, 1953, 1954, 618, + /* 2130 */ 1596, 613, 1596, 1596, 1596, 1950, 1596, 1596, 1596, 1596, + /* 2140 */ 1596, 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2150 */ 620, 1596, 1596, 1596, 1596, 1919, 1596, 619, 1596, 1596, + /* 2160 */ 1596, 1596, 1596, 1596, 1596, 1596, 1968, 1596, 1596, 1596, + /* 2170 */ 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, 1596, 1919, + /* 2180 */ 1949, 619, 1596, 1596, 1984, 1596, 1596, 298, 1951, 623, + /* 2190 */ 1953, 1954, 618, 1596, 613, 1596, 1596, 1950, 1596, 1596, + /* 2200 */ 1596, 1596, 1596, 1596, 1949, 1596, 1596, 1596, 1984, 1596, + /* 2210 */ 1596, 299, 1951, 623, 1953, 1954, 618, 1596, 613, 1596, + /* 2220 */ 1596, 1950, 1596, 1596, 1596, 1596, 1596, 1596, 1968, 1596, + /* 2230 */ 1596, 1596, 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, + /* 2240 */ 1596, 1919, 1596, 619, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2250 */ 1596, 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2260 */ 620, 1596, 1596, 1596, 1950, 1919, 1949, 619, 1596, 1596, + /* 2270 */ 1984, 1596, 1596, 305, 1951, 623, 1953, 1954, 618, 1596, + /* 2280 */ 613, 1596, 1950, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2290 */ 1949, 1596, 1596, 1596, 1984, 1968, 1596, 309, 1951, 623, + /* 2300 */ 1953, 1954, 618, 620, 613, 1596, 1596, 1596, 1919, 1596, + /* 2310 */ 619, 1596, 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2320 */ 1596, 620, 1596, 1596, 1596, 1596, 1919, 1596, 619, 1596, + /* 2330 */ 1596, 1596, 1596, 1949, 1596, 1596, 1596, 1984, 1596, 1596, + /* 2340 */ 301, 1951, 623, 1953, 1954, 618, 1596, 613, 1596, 1596, + /* 2350 */ 1596, 1949, 1596, 1596, 1950, 1984, 1596, 1596, 310, 1951, + /* 2360 */ 623, 1953, 1954, 618, 1596, 613, 1596, 1596, 1596, 1596, + /* 2370 */ 1596, 1596, 1950, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2380 */ 1596, 1596, 1596, 1596, 1596, 1968, 1596, 1596, 1596, 1596, + /* 2390 */ 1596, 1596, 1596, 620, 1596, 1596, 1596, 1596, 1919, 1596, + /* 2400 */ 619, 1596, 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2410 */ 1596, 620, 1596, 1596, 1596, 1950, 1919, 1596, 619, 1596, + /* 2420 */ 1596, 1596, 1596, 1949, 1596, 1596, 1596, 1984, 1596, 1596, + /* 2430 */ 302, 1951, 623, 1953, 1954, 618, 1596, 613, 1596, 1950, + /* 2440 */ 1596, 1949, 1596, 1596, 1596, 1984, 1968, 1596, 311, 1951, + /* 2450 */ 623, 1953, 1954, 618, 620, 613, 1596, 1596, 1596, 1919, + /* 2460 */ 1596, 619, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2470 */ 1968, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 620, 1596, + /* 2480 */ 1596, 1596, 1596, 1919, 1949, 619, 1596, 1596, 1984, 1596, + /* 2490 */ 1596, 303, 1951, 623, 1953, 1954, 618, 1596, 613, 1596, + /* 2500 */ 1596, 1596, 1950, 1596, 1596, 1596, 1596, 1596, 1949, 1596, + /* 2510 */ 1596, 1596, 1984, 1596, 1596, 316, 1951, 623, 1953, 1954, + /* 2520 */ 618, 1596, 613, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2530 */ 1596, 1596, 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2540 */ 1596, 620, 1596, 1596, 1596, 1596, 1919, 1596, 619, 1596, + /* 2550 */ 1596, 1596, 1596, 1596, 1596, 1596, 1950, 1596, 1596, 1596, + /* 2560 */ 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2570 */ 1596, 1949, 1596, 1596, 1596, 1984, 1596, 1596, 317, 1951, + /* 2580 */ 623, 1953, 1954, 618, 1596, 613, 1596, 1968, 1596, 1596, + /* 2590 */ 1596, 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, 1596, + /* 2600 */ 1919, 1596, 619, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2610 */ 1950, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2620 */ 1596, 1596, 1596, 1596, 1596, 1949, 1950, 1596, 1596, 1984, + /* 2630 */ 1596, 1596, 1962, 1951, 623, 1953, 1954, 618, 1596, 613, + /* 2640 */ 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 620, + /* 2650 */ 1596, 1596, 1596, 1596, 1919, 1596, 619, 1968, 1596, 1596, + /* 2660 */ 1596, 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, 1950, + /* 2670 */ 1919, 1596, 619, 1596, 1596, 1596, 1596, 1596, 1596, 1949, + /* 2680 */ 1596, 1596, 1596, 1984, 1596, 1596, 1961, 1951, 623, 1953, + /* 2690 */ 1954, 618, 1596, 613, 1596, 1949, 1596, 1596, 1596, 1984, + /* 2700 */ 1968, 1596, 1960, 1951, 623, 1953, 1954, 618, 620, 613, + /* 2710 */ 1596, 1596, 1596, 1919, 1596, 619, 1596, 1596, 1596, 1596, + /* 2720 */ 1596, 1596, 1596, 1596, 1596, 1596, 1950, 1596, 1596, 1596, + /* 2730 */ 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1949, 1596, + /* 2740 */ 1596, 1596, 1984, 1950, 1596, 328, 1951, 623, 1953, 1954, + /* 2750 */ 618, 1596, 613, 1596, 1596, 1596, 1596, 1968, 1596, 1596, + /* 2760 */ 1596, 1596, 1596, 1596, 1596, 620, 1596, 1596, 1596, 1950, + /* 2770 */ 1919, 1596, 619, 1596, 1968, 1596, 1596, 1596, 1596, 1596, + /* 2780 */ 1596, 1596, 620, 1596, 1596, 1596, 1596, 1919, 1596, 619, + /* 2790 */ 1596, 1596, 1596, 1596, 1596, 1949, 1596, 1596, 1596, 1984, + /* 2800 */ 1968, 1596, 329, 1951, 623, 1953, 1954, 618, 620, 613, + /* 2810 */ 1596, 1596, 1949, 1919, 1596, 619, 1984, 1596, 1596, 325, + /* 2820 */ 1951, 623, 1953, 1954, 618, 1596, 613, 1596, 1596, 1596, + /* 2830 */ 1950, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1949, 1596, + /* 2840 */ 1596, 1596, 1984, 1596, 1596, 330, 1951, 623, 1953, 1954, + /* 2850 */ 618, 1596, 613, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2860 */ 1596, 1968, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 620, + /* 2870 */ 1596, 1596, 1596, 1596, 1919, 1596, 619, 1596, 1596, 1596, + /* 2880 */ 1596, 1596, 1596, 1596, 1950, 1596, 1596, 1596, 1596, 1596, + /* 2890 */ 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 621, + /* 2900 */ 1596, 1596, 1596, 1984, 1596, 1596, 308, 1951, 623, 1953, + /* 2910 */ 1954, 618, 1596, 613, 1596, 1968, 1596, 1596, 1596, 1596, + /* 2920 */ 1596, 1596, 1596, 620, 1596, 1596, 1596, 1596, 1919, 1596, + /* 2930 */ 619, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2940 */ 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, 1596, + /* 2950 */ 1596, 1596, 1596, 1949, 1596, 1596, 1596, 1984, 1596, 1596, + /* 2960 */ 307, 1951, 623, 1953, 1954, 618, 1596, 613, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 385, 426, 347, 0, 328, 430, 330, 331, 415, 416, - /* 10 */ 355, 385, 12, 13, 14, 328, 355, 330, 331, 364, - /* 20 */ 20, 446, 22, 8, 9, 450, 451, 12, 13, 14, - /* 30 */ 15, 16, 20, 33, 339, 35, 324, 376, 377, 344, - /* 40 */ 21, 426, 8, 9, 370, 430, 12, 13, 14, 15, - /* 50 */ 16, 20, 426, 34, 380, 36, 430, 383, 58, 44, - /* 60 */ 445, 446, 385, 63, 355, 450, 451, 355, 369, 370, - /* 70 */ 70, 445, 446, 364, 426, 363, 450, 451, 430, 332, - /* 80 */ 368, 20, 370, 12, 13, 12, 13, 14, 15, 16, - /* 90 */ 323, 20, 325, 22, 446, 355, 96, 63, 450, 451, - /* 100 */ 1, 2, 362, 426, 33, 393, 35, 430, 4, 397, - /* 110 */ 363, 371, 400, 401, 402, 403, 404, 405, 118, 407, - /* 120 */ 58, 63, 445, 446, 412, 33, 414, 450, 451, 58, - /* 130 */ 418, 419, 132, 133, 63, 14, 15, 16, 62, 105, - /* 140 */ 48, 70, 399, 332, 432, 357, 54, 55, 56, 57, - /* 150 */ 58, 366, 440, 338, 369, 370, 368, 96, 96, 355, - /* 160 */ 98, 161, 162, 132, 133, 355, 362, 96, 425, 422, - /* 170 */ 423, 424, 362, 426, 427, 371, 361, 430, 20, 128, - /* 180 */ 180, 371, 182, 372, 396, 397, 398, 95, 20, 118, - /* 190 */ 98, 363, 445, 446, 160, 407, 97, 450, 451, 324, - /* 200 */ 96, 14, 374, 132, 133, 205, 206, 20, 208, 209, + /* 0 */ 331, 350, 333, 334, 359, 331, 324, 333, 334, 358, + /* 10 */ 391, 341, 12, 13, 14, 360, 358, 326, 367, 328, + /* 20 */ 20, 20, 22, 8, 9, 355, 371, 12, 13, 14, + /* 30 */ 15, 16, 20, 33, 364, 35, 0, 379, 380, 20, + /* 40 */ 360, 369, 8, 9, 372, 373, 12, 13, 14, 15, + /* 50 */ 16, 371, 335, 366, 399, 400, 401, 335, 58, 44, + /* 60 */ 14, 2, 0, 63, 377, 410, 20, 8, 9, 387, + /* 70 */ 70, 12, 13, 14, 15, 16, 357, 327, 358, 399, + /* 80 */ 400, 401, 4, 12, 13, 365, 20, 360, 366, 370, + /* 90 */ 410, 20, 375, 22, 374, 95, 332, 63, 371, 335, + /* 100 */ 336, 65, 66, 67, 33, 350, 35, 95, 358, 73, + /* 110 */ 74, 429, 14, 358, 78, 433, 366, 117, 20, 83, + /* 120 */ 84, 371, 367, 373, 62, 89, 399, 400, 20, 58, + /* 130 */ 448, 449, 132, 133, 63, 453, 454, 410, 104, 332, + /* 140 */ 342, 70, 335, 336, 335, 347, 396, 425, 426, 427, + /* 150 */ 400, 429, 430, 403, 404, 405, 406, 407, 408, 429, + /* 160 */ 410, 161, 162, 433, 163, 415, 95, 417, 65, 66, + /* 170 */ 67, 421, 422, 95, 62, 366, 73, 74, 448, 449, + /* 180 */ 180, 78, 182, 453, 454, 435, 83, 84, 117, 414, + /* 190 */ 360, 416, 89, 443, 160, 127, 12, 13, 14, 15, + /* 200 */ 16, 371, 4, 132, 133, 205, 206, 161, 208, 209, /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 220 */ 220, 221, 222, 223, 224, 225, 226, 65, 66, 67, - /* 230 */ 426, 20, 161, 162, 430, 73, 74, 20, 187, 188, - /* 240 */ 78, 79, 191, 368, 193, 321, 84, 85, 4, 445, - /* 250 */ 446, 180, 90, 182, 450, 451, 164, 165, 324, 167, - /* 260 */ 80, 166, 170, 205, 205, 231, 232, 233, 234, 235, - /* 270 */ 236, 237, 238, 239, 240, 241, 205, 206, 186, 208, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 161, 162, 399, + /* 230 */ 400, 20, 161, 162, 425, 426, 427, 350, 429, 430, + /* 240 */ 410, 43, 433, 45, 46, 358, 95, 20, 358, 22, + /* 250 */ 37, 180, 205, 182, 367, 187, 188, 448, 449, 191, + /* 260 */ 79, 193, 453, 454, 374, 231, 232, 233, 234, 235, + /* 270 */ 236, 237, 238, 239, 240, 241, 205, 206, 51, 208, /* 280 */ 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 354, 228, - /* 300 */ 12, 13, 368, 341, 342, 244, 244, 96, 20, 385, - /* 310 */ 22, 367, 253, 254, 255, 256, 257, 137, 138, 161, - /* 320 */ 162, 33, 324, 35, 109, 110, 111, 112, 113, 114, - /* 330 */ 115, 116, 117, 118, 119, 20, 121, 122, 123, 124, - /* 340 */ 125, 126, 175, 336, 249, 250, 58, 20, 244, 22, - /* 350 */ 426, 63, 329, 355, 430, 332, 333, 350, 70, 341, - /* 360 */ 342, 363, 35, 196, 197, 358, 368, 44, 370, 445, - /* 370 */ 446, 12, 13, 14, 450, 451, 2, 355, 51, 20, - /* 380 */ 163, 22, 8, 9, 96, 363, 12, 13, 14, 15, - /* 390 */ 16, 393, 33, 329, 35, 397, 332, 333, 400, 401, - /* 400 */ 402, 403, 404, 405, 332, 407, 118, 324, 8, 9, - /* 410 */ 399, 96, 12, 13, 14, 15, 16, 58, 346, 127, - /* 420 */ 132, 133, 108, 163, 402, 353, 8, 9, 168, 70, - /* 430 */ 12, 13, 14, 15, 16, 363, 425, 35, 355, 441, - /* 440 */ 442, 127, 128, 129, 130, 131, 363, 336, 370, 161, - /* 450 */ 162, 368, 96, 370, 426, 96, 242, 243, 430, 8, - /* 460 */ 9, 383, 20, 12, 13, 14, 15, 16, 180, 358, - /* 470 */ 182, 20, 70, 445, 446, 324, 393, 118, 450, 451, - /* 480 */ 397, 189, 190, 400, 401, 402, 403, 404, 405, 245, - /* 490 */ 407, 132, 133, 205, 206, 96, 208, 209, 210, 211, + /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 3, 228, + /* 300 */ 12, 13, 255, 256, 257, 258, 259, 228, 20, 230, + /* 310 */ 22, 98, 20, 100, 101, 20, 103, 335, 137, 138, + /* 320 */ 107, 33, 244, 35, 108, 109, 110, 111, 112, 113, + /* 330 */ 114, 115, 116, 117, 118, 0, 120, 121, 122, 123, + /* 340 */ 124, 125, 129, 132, 133, 58, 58, 58, 366, 58, + /* 350 */ 429, 63, 126, 63, 433, 339, 21, 228, 70, 24, + /* 360 */ 25, 26, 27, 28, 29, 30, 31, 32, 358, 353, + /* 370 */ 449, 12, 13, 14, 453, 454, 414, 361, 416, 20, + /* 380 */ 387, 22, 95, 95, 97, 94, 97, 95, 97, 95, + /* 390 */ 380, 96, 33, 21, 35, 244, 24, 25, 26, 27, + /* 400 */ 28, 29, 30, 31, 32, 117, 424, 425, 426, 427, + /* 410 */ 95, 429, 430, 33, 373, 189, 190, 58, 373, 429, + /* 420 */ 132, 133, 429, 433, 383, 341, 433, 386, 48, 70, + /* 430 */ 20, 386, 22, 166, 54, 55, 56, 57, 58, 449, + /* 440 */ 22, 448, 449, 453, 454, 35, 453, 454, 364, 161, + /* 450 */ 162, 126, 127, 35, 95, 0, 131, 20, 163, 8, + /* 460 */ 9, 51, 358, 12, 13, 14, 15, 16, 180, 365, + /* 470 */ 182, 20, 20, 179, 94, 181, 117, 97, 374, 24, + /* 480 */ 25, 26, 27, 28, 29, 30, 31, 32, 70, 402, + /* 490 */ 327, 132, 133, 205, 206, 205, 208, 209, 210, 211, /* 500 */ 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, - /* 510 */ 222, 223, 224, 225, 226, 97, 4, 324, 324, 368, - /* 520 */ 161, 162, 0, 21, 45, 46, 24, 25, 26, 27, - /* 530 */ 28, 29, 30, 31, 32, 452, 453, 127, 128, 180, - /* 540 */ 228, 182, 230, 21, 0, 324, 24, 25, 26, 27, - /* 550 */ 28, 29, 30, 31, 32, 43, 20, 45, 46, 244, - /* 560 */ 62, 368, 368, 163, 205, 206, 160, 208, 209, 210, + /* 510 */ 222, 223, 224, 225, 226, 428, 249, 250, 251, 14, + /* 520 */ 161, 162, 8, 9, 160, 20, 12, 13, 14, 15, + /* 530 */ 16, 244, 95, 429, 371, 117, 244, 433, 244, 180, + /* 540 */ 175, 182, 20, 358, 164, 165, 0, 167, 418, 419, + /* 550 */ 170, 366, 448, 449, 358, 387, 20, 453, 454, 244, + /* 560 */ 327, 196, 197, 367, 205, 206, 186, 208, 209, 210, /* 570 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - /* 580 */ 221, 222, 223, 224, 225, 226, 12, 13, 265, 368, - /* 590 */ 19, 4, 20, 20, 20, 22, 22, 363, 188, 332, - /* 600 */ 244, 191, 332, 193, 33, 14, 19, 33, 374, 35, - /* 610 */ 324, 20, 96, 346, 163, 107, 346, 332, 324, 48, - /* 620 */ 33, 228, 332, 353, 51, 54, 55, 56, 57, 58, - /* 630 */ 363, 346, 58, 363, 347, 48, 338, 231, 355, 70, - /* 640 */ 53, 355, 355, 244, 70, 58, 70, 241, 363, 363, - /* 650 */ 352, 364, 108, 363, 368, 0, 370, 12, 13, 361, - /* 660 */ 377, 0, 368, 332, 324, 20, 95, 22, 0, 98, - /* 670 */ 96, 127, 128, 129, 130, 131, 356, 346, 33, 393, - /* 680 */ 35, 324, 95, 397, 356, 98, 400, 401, 402, 403, - /* 690 */ 404, 405, 118, 407, 363, 179, 410, 181, 412, 413, - /* 700 */ 414, 347, 131, 58, 418, 419, 132, 133, 368, 355, - /* 710 */ 389, 421, 422, 423, 424, 70, 426, 427, 364, 356, - /* 720 */ 65, 66, 67, 62, 58, 368, 14, 332, 73, 74, - /* 730 */ 62, 324, 20, 78, 79, 161, 162, 324, 167, 84, - /* 740 */ 85, 96, 0, 8, 9, 90, 20, 12, 13, 14, - /* 750 */ 15, 16, 161, 20, 180, 184, 182, 186, 363, 348, - /* 760 */ 244, 95, 351, 118, 98, 357, 24, 25, 26, 27, - /* 770 */ 28, 29, 30, 31, 32, 368, 368, 132, 133, 205, - /* 780 */ 206, 368, 208, 209, 210, 211, 212, 213, 214, 215, + /* 580 */ 221, 222, 223, 224, 225, 226, 12, 13, 335, 335, + /* 590 */ 405, 358, 372, 373, 20, 231, 22, 429, 180, 366, + /* 600 */ 182, 433, 349, 349, 371, 241, 373, 33, 62, 35, + /* 610 */ 96, 14, 15, 16, 163, 163, 448, 449, 358, 366, + /* 620 */ 366, 453, 454, 205, 206, 365, 387, 344, 345, 396, + /* 630 */ 3, 335, 58, 400, 374, 39, 403, 404, 405, 406, + /* 640 */ 407, 408, 0, 410, 70, 349, 413, 402, 415, 416, + /* 650 */ 417, 327, 356, 402, 421, 422, 387, 12, 13, 20, + /* 660 */ 4, 21, 366, 335, 327, 20, 161, 22, 429, 95, + /* 670 */ 1, 2, 433, 428, 34, 19, 36, 349, 33, 428, + /* 680 */ 35, 244, 358, 339, 356, 366, 35, 448, 449, 33, + /* 690 */ 366, 117, 453, 454, 366, 371, 377, 373, 429, 344, + /* 700 */ 345, 20, 433, 58, 48, 361, 132, 133, 371, 53, + /* 710 */ 413, 335, 106, 416, 58, 70, 327, 448, 449, 327, + /* 720 */ 396, 70, 453, 454, 400, 349, 366, 403, 404, 405, + /* 730 */ 406, 407, 408, 335, 410, 161, 162, 377, 62, 415, + /* 740 */ 95, 417, 366, 335, 335, 421, 422, 349, 351, 107, + /* 750 */ 94, 354, 368, 97, 180, 371, 182, 349, 349, 107, + /* 760 */ 371, 327, 117, 371, 366, 96, 0, 443, 126, 127, + /* 770 */ 128, 129, 130, 131, 366, 366, 70, 132, 133, 205, + /* 780 */ 206, 129, 208, 209, 210, 211, 212, 213, 214, 215, /* 790 */ 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, - /* 800 */ 226, 18, 108, 20, 396, 397, 161, 162, 324, 363, - /* 810 */ 27, 399, 411, 30, 413, 407, 33, 422, 423, 424, - /* 820 */ 374, 426, 427, 355, 130, 180, 0, 182, 324, 0, - /* 830 */ 324, 48, 97, 50, 2, 410, 53, 425, 413, 371, - /* 840 */ 8, 9, 324, 324, 12, 13, 14, 15, 16, 356, - /* 850 */ 205, 206, 368, 208, 209, 210, 211, 212, 213, 214, + /* 800 */ 226, 18, 163, 20, 335, 371, 161, 162, 327, 368, + /* 810 */ 27, 368, 371, 30, 371, 49, 33, 351, 349, 70, + /* 820 */ 354, 346, 42, 348, 44, 180, 0, 182, 327, 327, + /* 830 */ 327, 48, 44, 50, 2, 366, 53, 45, 46, 243, + /* 840 */ 8, 9, 0, 0, 12, 13, 14, 15, 16, 20, + /* 850 */ 205, 206, 371, 208, 209, 210, 211, 212, 213, 214, /* 860 */ 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - /* 870 */ 225, 226, 368, 161, 368, 8, 9, 48, 95, 12, - /* 880 */ 13, 14, 15, 16, 20, 365, 368, 368, 368, 163, - /* 890 */ 107, 65, 66, 67, 68, 69, 163, 71, 72, 73, + /* 870 */ 225, 226, 371, 371, 371, 8, 9, 94, 359, 12, + /* 880 */ 13, 14, 15, 16, 96, 99, 242, 243, 102, 106, + /* 890 */ 263, 65, 66, 67, 68, 69, 4, 71, 72, 73, /* 900 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 910 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 136, - /* 920 */ 324, 324, 139, 140, 141, 142, 143, 144, 145, 146, - /* 930 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 3, - /* 940 */ 157, 158, 159, 18, 411, 332, 413, 332, 23, 8, - /* 950 */ 9, 44, 355, 12, 13, 14, 15, 16, 332, 346, - /* 960 */ 363, 346, 37, 38, 368, 368, 41, 370, 382, 385, - /* 970 */ 384, 3, 346, 44, 324, 324, 363, 382, 363, 384, - /* 980 */ 324, 332, 1, 2, 59, 60, 61, 0, 20, 363, - /* 990 */ 393, 332, 332, 365, 397, 346, 368, 400, 401, 402, - /* 1000 */ 403, 404, 405, 39, 407, 355, 346, 0, 356, 412, - /* 1010 */ 426, 414, 363, 363, 430, 418, 419, 344, 368, 368, - /* 1020 */ 370, 96, 363, 363, 368, 161, 97, 332, 348, 445, - /* 1030 */ 446, 351, 332, 100, 450, 451, 103, 440, 357, 172, - /* 1040 */ 356, 346, 332, 393, 58, 100, 346, 397, 103, 368, - /* 1050 */ 400, 401, 402, 403, 404, 405, 346, 407, 363, 134, - /* 1060 */ 332, 324, 412, 363, 414, 97, 12, 13, 418, 419, - /* 1070 */ 343, 356, 345, 363, 346, 35, 22, 396, 397, 429, - /* 1080 */ 332, 422, 423, 424, 98, 426, 427, 33, 407, 35, - /* 1090 */ 325, 363, 355, 22, 346, 108, 107, 172, 173, 174, - /* 1100 */ 363, 42, 177, 44, 324, 368, 35, 370, 42, 378, - /* 1110 */ 44, 363, 58, 443, 127, 128, 129, 130, 131, 35, - /* 1120 */ 195, 357, 332, 198, 70, 200, 201, 202, 203, 204, - /* 1130 */ 393, 163, 368, 266, 397, 355, 346, 400, 401, 402, - /* 1140 */ 403, 404, 405, 363, 407, 156, 0, 96, 368, 412, - /* 1150 */ 370, 414, 454, 363, 0, 418, 419, 106, 151, 334, - /* 1160 */ 396, 397, 398, 385, 100, 332, 429, 103, 22, 244, - /* 1170 */ 263, 407, 118, 393, 332, 324, 22, 397, 44, 346, - /* 1180 */ 400, 401, 402, 403, 404, 405, 332, 407, 346, 332, - /* 1190 */ 0, 47, 412, 437, 414, 355, 363, 261, 418, 419, - /* 1200 */ 346, 332, 334, 346, 426, 363, 355, 243, 430, 429, - /* 1210 */ 132, 133, 22, 100, 363, 346, 103, 363, 324, 368, - /* 1220 */ 363, 370, 182, 445, 446, 44, 44, 44, 450, 451, - /* 1230 */ 44, 97, 363, 44, 180, 44, 182, 44, 378, 331, - /* 1240 */ 96, 367, 428, 378, 393, 447, 35, 420, 397, 355, - /* 1250 */ 44, 400, 401, 402, 403, 404, 405, 363, 407, 205, - /* 1260 */ 206, 324, 368, 412, 370, 414, 182, 44, 44, 418, - /* 1270 */ 419, 217, 218, 219, 220, 221, 222, 223, 97, 97, - /* 1280 */ 97, 70, 246, 97, 0, 324, 97, 393, 97, 13, - /* 1290 */ 97, 397, 355, 44, 400, 401, 402, 403, 404, 405, - /* 1300 */ 363, 407, 431, 97, 44, 368, 412, 370, 414, 395, - /* 1310 */ 48, 35, 418, 419, 13, 394, 355, 44, 44, 22, - /* 1320 */ 97, 97, 387, 178, 363, 42, 375, 20, 378, 368, - /* 1330 */ 393, 370, 35, 49, 397, 37, 35, 400, 401, 402, - /* 1340 */ 403, 404, 405, 324, 407, 375, 97, 160, 373, 412, - /* 1350 */ 20, 414, 332, 332, 393, 418, 419, 97, 397, 375, - /* 1360 */ 94, 400, 401, 402, 403, 404, 405, 70, 407, 373, - /* 1370 */ 97, 97, 373, 340, 355, 414, 332, 332, 332, 418, - /* 1380 */ 419, 20, 363, 326, 326, 20, 324, 368, 391, 370, - /* 1390 */ 338, 338, 370, 20, 333, 20, 386, 99, 338, 101, - /* 1400 */ 102, 333, 104, 338, 332, 338, 108, 338, 338, 52, - /* 1410 */ 335, 335, 393, 326, 324, 118, 397, 355, 355, 400, - /* 1420 */ 401, 402, 403, 404, 405, 363, 407, 332, 130, 355, - /* 1430 */ 368, 326, 370, 414, 355, 355, 355, 418, 419, 194, - /* 1440 */ 355, 355, 392, 355, 355, 355, 355, 355, 336, 185, - /* 1450 */ 368, 336, 332, 363, 252, 393, 251, 258, 368, 397, - /* 1460 */ 370, 378, 400, 401, 402, 403, 404, 405, 391, 407, - /* 1470 */ 370, 378, 368, 324, 436, 368, 414, 180, 381, 182, - /* 1480 */ 418, 419, 381, 393, 390, 436, 368, 397, 171, 368, - /* 1490 */ 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, - /* 1500 */ 260, 259, 205, 206, 355, 247, 395, 264, 267, 262, - /* 1510 */ 455, 438, 363, 243, 363, 20, 399, 368, 333, 370, - /* 1520 */ 332, 381, 381, 368, 368, 368, 368, 368, 336, 368, - /* 1530 */ 336, 324, 165, 434, 436, 439, 433, 435, 363, 379, - /* 1540 */ 336, 96, 393, 417, 96, 368, 397, 351, 345, 400, - /* 1550 */ 401, 402, 403, 404, 405, 449, 407, 448, 332, 324, - /* 1560 */ 359, 336, 355, 414, 36, 327, 388, 349, 419, 349, - /* 1570 */ 363, 349, 326, 337, 322, 368, 0, 370, 0, 384, - /* 1580 */ 187, 0, 0, 42, 0, 324, 35, 199, 35, 35, - /* 1590 */ 355, 35, 0, 199, 35, 35, 199, 0, 363, 199, - /* 1600 */ 393, 0, 35, 368, 397, 370, 0, 400, 401, 402, - /* 1610 */ 403, 404, 405, 0, 407, 22, 355, 182, 35, 180, - /* 1620 */ 0, 0, 176, 0, 363, 175, 0, 47, 393, 368, - /* 1630 */ 0, 370, 397, 0, 0, 400, 401, 402, 403, 404, - /* 1640 */ 405, 42, 407, 0, 0, 0, 0, 0, 0, 442, - /* 1650 */ 0, 0, 151, 35, 393, 324, 0, 151, 397, 0, - /* 1660 */ 0, 400, 401, 402, 403, 404, 405, 0, 407, 0, - /* 1670 */ 0, 0, 0, 0, 324, 22, 0, 42, 0, 444, - /* 1680 */ 0, 0, 0, 0, 0, 0, 355, 0, 0, 0, - /* 1690 */ 0, 360, 135, 0, 363, 0, 0, 0, 0, 368, - /* 1700 */ 35, 370, 0, 0, 0, 355, 0, 58, 42, 47, - /* 1710 */ 58, 47, 44, 363, 453, 14, 39, 0, 368, 14, - /* 1720 */ 370, 47, 0, 0, 393, 0, 40, 0, 397, 171, - /* 1730 */ 39, 400, 401, 402, 403, 404, 405, 324, 407, 39, - /* 1740 */ 0, 0, 0, 393, 0, 35, 39, 397, 0, 35, - /* 1750 */ 400, 401, 402, 403, 404, 405, 48, 407, 324, 409, - /* 1760 */ 48, 0, 0, 64, 35, 39, 48, 39, 355, 35, - /* 1770 */ 39, 0, 48, 360, 0, 0, 363, 0, 35, 22, - /* 1780 */ 324, 368, 0, 370, 105, 35, 35, 22, 35, 355, - /* 1790 */ 35, 44, 22, 44, 360, 35, 0, 363, 35, 35, - /* 1800 */ 22, 324, 368, 0, 370, 22, 393, 0, 50, 22, - /* 1810 */ 397, 355, 103, 400, 401, 402, 403, 404, 405, 363, - /* 1820 */ 407, 35, 0, 324, 368, 35, 370, 393, 0, 35, - /* 1830 */ 0, 397, 355, 22, 400, 401, 402, 403, 404, 405, - /* 1840 */ 363, 407, 97, 20, 324, 368, 0, 370, 96, 393, - /* 1850 */ 35, 0, 0, 397, 355, 22, 400, 401, 402, 403, - /* 1860 */ 404, 405, 363, 407, 0, 44, 3, 368, 96, 370, - /* 1870 */ 393, 44, 248, 97, 397, 355, 163, 400, 401, 402, - /* 1880 */ 403, 404, 405, 363, 407, 165, 97, 324, 368, 96, - /* 1890 */ 370, 96, 393, 163, 183, 163, 397, 97, 97, 400, - /* 1900 */ 401, 402, 403, 404, 405, 169, 407, 47, 96, 96, - /* 1910 */ 44, 44, 47, 393, 96, 324, 3, 397, 355, 44, - /* 1920 */ 400, 401, 402, 403, 404, 405, 363, 407, 97, 97, - /* 1930 */ 35, 368, 35, 370, 35, 35, 35, 35, 97, 97, - /* 1940 */ 47, 324, 248, 47, 0, 0, 355, 44, 0, 96, - /* 1950 */ 35, 97, 97, 96, 363, 35, 393, 242, 192, 368, - /* 1960 */ 397, 370, 0, 400, 401, 402, 403, 404, 405, 248, - /* 1970 */ 407, 166, 355, 96, 96, 0, 39, 96, 164, 106, - /* 1980 */ 363, 47, 44, 227, 393, 368, 2, 370, 397, 22, - /* 1990 */ 205, 400, 401, 402, 403, 404, 405, 229, 407, 324, - /* 2000 */ 227, 97, 96, 96, 47, 97, 96, 47, 97, 97, - /* 2010 */ 393, 207, 96, 96, 397, 22, 324, 400, 401, 402, - /* 2020 */ 403, 404, 405, 97, 407, 96, 107, 35, 96, 35, - /* 2030 */ 355, 35, 97, 97, 35, 96, 96, 35, 363, 97, - /* 2040 */ 97, 35, 96, 368, 120, 370, 96, 355, 120, 22, - /* 2050 */ 120, 120, 96, 35, 96, 363, 44, 108, 22, 324, - /* 2060 */ 368, 96, 370, 35, 64, 35, 63, 35, 393, 35, - /* 2070 */ 35, 70, 397, 35, 35, 400, 401, 402, 403, 404, - /* 2080 */ 405, 35, 407, 324, 35, 393, 35, 44, 93, 397, - /* 2090 */ 355, 35, 400, 401, 402, 403, 404, 405, 363, 407, - /* 2100 */ 35, 22, 35, 368, 22, 370, 35, 35, 22, 35, - /* 2110 */ 70, 35, 35, 35, 355, 35, 35, 0, 35, 35, - /* 2120 */ 0, 0, 363, 48, 39, 35, 324, 368, 393, 370, - /* 2130 */ 39, 39, 397, 35, 0, 400, 401, 402, 403, 404, - /* 2140 */ 405, 35, 407, 324, 48, 48, 39, 48, 0, 35, - /* 2150 */ 35, 0, 393, 22, 21, 20, 397, 355, 21, 400, - /* 2160 */ 401, 402, 403, 404, 405, 363, 407, 22, 22, 324, - /* 2170 */ 368, 456, 370, 456, 355, 456, 456, 456, 456, 456, - /* 2180 */ 456, 456, 363, 456, 456, 456, 324, 368, 456, 370, - /* 2190 */ 456, 456, 456, 456, 456, 393, 456, 456, 456, 397, - /* 2200 */ 355, 456, 400, 401, 402, 403, 404, 405, 363, 407, - /* 2210 */ 456, 456, 393, 368, 456, 370, 397, 355, 456, 400, - /* 2220 */ 401, 402, 403, 404, 405, 363, 407, 456, 456, 324, - /* 2230 */ 368, 456, 370, 456, 456, 456, 456, 456, 393, 456, - /* 2240 */ 456, 456, 397, 456, 456, 400, 401, 402, 403, 404, - /* 2250 */ 405, 324, 407, 456, 456, 393, 456, 456, 456, 397, - /* 2260 */ 355, 456, 400, 401, 402, 403, 404, 405, 363, 407, - /* 2270 */ 456, 456, 456, 368, 456, 370, 456, 456, 456, 456, - /* 2280 */ 456, 456, 355, 456, 456, 456, 456, 456, 456, 456, - /* 2290 */ 363, 456, 456, 456, 456, 368, 456, 370, 393, 456, - /* 2300 */ 456, 456, 397, 456, 456, 400, 401, 402, 403, 404, - /* 2310 */ 405, 456, 407, 324, 456, 456, 456, 456, 456, 456, - /* 2320 */ 393, 456, 456, 456, 397, 456, 456, 400, 401, 402, - /* 2330 */ 403, 404, 405, 456, 407, 324, 456, 456, 456, 456, - /* 2340 */ 456, 456, 456, 456, 355, 456, 456, 456, 456, 456, - /* 2350 */ 456, 456, 363, 456, 456, 456, 456, 368, 456, 370, - /* 2360 */ 456, 456, 456, 456, 456, 456, 355, 456, 456, 456, - /* 2370 */ 456, 456, 456, 456, 363, 456, 456, 456, 456, 368, - /* 2380 */ 456, 370, 393, 456, 456, 456, 397, 456, 456, 400, - /* 2390 */ 401, 402, 403, 404, 405, 456, 407, 324, 456, 456, - /* 2400 */ 456, 456, 456, 456, 393, 456, 456, 456, 397, 456, - /* 2410 */ 456, 400, 401, 402, 403, 404, 405, 324, 407, 456, - /* 2420 */ 456, 456, 456, 456, 456, 456, 456, 456, 355, 456, - /* 2430 */ 456, 456, 456, 456, 456, 456, 363, 456, 456, 456, - /* 2440 */ 456, 368, 456, 370, 456, 456, 456, 456, 355, 456, - /* 2450 */ 456, 456, 456, 456, 456, 456, 363, 456, 456, 456, - /* 2460 */ 324, 368, 456, 370, 456, 456, 393, 456, 456, 456, - /* 2470 */ 397, 456, 456, 400, 401, 402, 403, 404, 405, 456, - /* 2480 */ 407, 456, 456, 456, 456, 456, 393, 456, 456, 456, - /* 2490 */ 397, 355, 456, 400, 401, 402, 403, 404, 405, 363, - /* 2500 */ 407, 456, 456, 456, 368, 456, 370, 456, 456, 456, - /* 2510 */ 456, 456, 456, 456, 456, 456, 456, 456, 456, 456, - /* 2520 */ 456, 456, 456, 456, 456, 456, 456, 456, 456, 393, - /* 2530 */ 456, 456, 456, 397, 456, 456, 400, 401, 402, 403, - /* 2540 */ 404, 405, 456, 407, + /* 910 */ 84, 85, 86, 87, 88, 89, 90, 91, 42, 136, + /* 920 */ 44, 35, 139, 140, 141, 142, 143, 144, 145, 146, + /* 930 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 106, + /* 940 */ 157, 158, 159, 18, 44, 387, 8, 9, 23, 107, + /* 950 */ 12, 13, 14, 15, 16, 99, 99, 327, 102, 102, + /* 960 */ 327, 327, 37, 38, 99, 328, 41, 102, 126, 127, + /* 970 */ 128, 129, 130, 131, 359, 8, 9, 327, 335, 12, + /* 980 */ 13, 14, 15, 16, 59, 60, 61, 429, 358, 156, + /* 990 */ 161, 433, 349, 335, 151, 44, 366, 327, 327, 327, + /* 1000 */ 95, 371, 22, 373, 371, 371, 448, 449, 358, 366, + /* 1010 */ 105, 453, 454, 335, 163, 35, 366, 335, 335, 168, + /* 1020 */ 95, 371, 327, 373, 366, 0, 396, 349, 44, 44, + /* 1030 */ 400, 349, 349, 403, 404, 405, 406, 407, 408, 172, + /* 1040 */ 410, 371, 371, 371, 366, 415, 396, 417, 366, 366, + /* 1050 */ 400, 421, 422, 403, 404, 405, 406, 407, 408, 134, + /* 1060 */ 410, 35, 432, 96, 335, 415, 371, 417, 182, 327, + /* 1070 */ 107, 421, 422, 48, 327, 132, 133, 327, 349, 359, + /* 1080 */ 96, 96, 432, 425, 426, 427, 359, 429, 430, 126, + /* 1090 */ 127, 128, 129, 130, 131, 366, 359, 172, 173, 174, + /* 1100 */ 358, 163, 177, 335, 359, 335, 347, 381, 366, 446, + /* 1110 */ 335, 0, 327, 371, 44, 373, 327, 349, 371, 349, + /* 1120 */ 195, 371, 457, 198, 349, 200, 201, 202, 203, 204, + /* 1130 */ 335, 1, 2, 22, 366, 268, 366, 245, 396, 335, + /* 1140 */ 335, 366, 400, 358, 349, 403, 404, 405, 406, 407, + /* 1150 */ 408, 366, 410, 349, 349, 327, 371, 415, 373, 417, + /* 1160 */ 371, 366, 0, 421, 422, 0, 96, 267, 327, 244, + /* 1170 */ 366, 366, 8, 9, 432, 44, 12, 13, 14, 15, + /* 1180 */ 16, 396, 335, 337, 22, 400, 358, 22, 403, 404, + /* 1190 */ 405, 406, 407, 408, 366, 410, 349, 440, 47, 371, + /* 1200 */ 415, 373, 417, 44, 44, 44, 421, 422, 182, 337, + /* 1210 */ 44, 44, 371, 366, 334, 44, 265, 432, 44, 12, + /* 1220 */ 13, 44, 327, 44, 396, 13, 44, 96, 400, 22, + /* 1230 */ 44, 403, 404, 405, 406, 407, 408, 13, 410, 35, + /* 1240 */ 33, 358, 35, 415, 381, 417, 95, 35, 370, 421, + /* 1250 */ 422, 381, 431, 358, 423, 96, 96, 96, 450, 35, + /* 1260 */ 432, 366, 96, 96, 434, 58, 371, 96, 373, 246, + /* 1270 */ 96, 398, 397, 96, 70, 96, 327, 70, 96, 48, + /* 1280 */ 178, 42, 96, 389, 378, 20, 378, 160, 381, 376, + /* 1290 */ 20, 396, 335, 335, 378, 400, 376, 376, 403, 404, + /* 1300 */ 405, 406, 407, 408, 93, 410, 335, 358, 343, 335, + /* 1310 */ 415, 335, 417, 20, 20, 366, 421, 422, 329, 329, + /* 1320 */ 371, 393, 373, 341, 117, 373, 20, 341, 336, 20, + /* 1330 */ 388, 336, 341, 335, 341, 341, 52, 341, 341, 338, + /* 1340 */ 338, 329, 329, 335, 371, 396, 358, 194, 371, 400, + /* 1350 */ 395, 392, 403, 404, 405, 406, 407, 408, 358, 410, + /* 1360 */ 339, 358, 327, 358, 415, 358, 417, 358, 358, 358, + /* 1370 */ 421, 422, 358, 358, 358, 185, 393, 373, 339, 335, + /* 1380 */ 381, 260, 254, 439, 439, 253, 171, 180, 442, 182, + /* 1390 */ 371, 371, 438, 358, 441, 371, 262, 261, 381, 371, + /* 1400 */ 439, 366, 384, 437, 247, 436, 371, 269, 373, 266, + /* 1410 */ 264, 384, 205, 206, 458, 243, 452, 366, 20, 451, + /* 1420 */ 398, 335, 339, 336, 217, 218, 219, 220, 221, 222, + /* 1430 */ 223, 396, 402, 384, 371, 400, 327, 371, 403, 404, + /* 1440 */ 405, 406, 407, 408, 371, 410, 371, 371, 384, 371, + /* 1450 */ 415, 165, 417, 382, 339, 354, 421, 422, 339, 366, + /* 1460 */ 95, 420, 95, 371, 348, 362, 36, 358, 335, 330, + /* 1470 */ 329, 390, 352, 394, 339, 366, 385, 352, 352, 327, + /* 1480 */ 371, 385, 373, 325, 340, 0, 0, 187, 0, 0, + /* 1490 */ 42, 0, 35, 35, 199, 35, 35, 199, 0, 35, + /* 1500 */ 35, 199, 0, 199, 0, 396, 35, 0, 0, 400, + /* 1510 */ 358, 35, 403, 404, 405, 406, 407, 408, 366, 410, + /* 1520 */ 182, 22, 327, 371, 180, 373, 417, 176, 175, 0, + /* 1530 */ 421, 422, 0, 0, 0, 0, 0, 47, 0, 42, + /* 1540 */ 0, 0, 0, 0, 0, 0, 0, 0, 396, 35, + /* 1550 */ 327, 151, 400, 358, 0, 403, 404, 405, 406, 407, + /* 1560 */ 408, 366, 410, 151, 0, 0, 371, 0, 373, 417, + /* 1570 */ 42, 0, 0, 421, 422, 0, 0, 0, 0, 0, + /* 1580 */ 0, 358, 0, 0, 0, 0, 0, 0, 0, 366, + /* 1590 */ 0, 396, 0, 0, 371, 400, 373, 0, 403, 404, + /* 1600 */ 405, 406, 407, 408, 0, 410, 0, 135, 0, 35, + /* 1610 */ 0, 22, 417, 58, 0, 58, 421, 422, 0, 396, + /* 1620 */ 0, 14, 19, 400, 0, 44, 403, 404, 405, 406, + /* 1630 */ 407, 408, 409, 410, 411, 412, 33, 42, 327, 39, + /* 1640 */ 14, 47, 40, 47, 39, 47, 0, 0, 0, 171, + /* 1650 */ 39, 48, 0, 0, 0, 64, 0, 54, 55, 56, + /* 1660 */ 57, 58, 0, 35, 48, 0, 35, 48, 39, 358, + /* 1670 */ 0, 39, 35, 48, 39, 0, 35, 366, 48, 39, + /* 1680 */ 0, 327, 371, 0, 373, 0, 0, 0, 22, 44, + /* 1690 */ 35, 22, 35, 35, 35, 35, 0, 94, 327, 35, + /* 1700 */ 97, 22, 35, 104, 0, 44, 35, 396, 0, 35, + /* 1710 */ 22, 400, 358, 50, 403, 404, 405, 406, 407, 408, + /* 1720 */ 366, 410, 22, 102, 0, 371, 35, 373, 0, 358, + /* 1730 */ 35, 0, 22, 130, 20, 96, 35, 366, 35, 95, + /* 1740 */ 192, 0, 371, 0, 373, 183, 35, 22, 0, 0, + /* 1750 */ 396, 44, 327, 3, 400, 444, 445, 403, 404, 405, + /* 1760 */ 406, 407, 408, 248, 410, 252, 165, 396, 327, 96, + /* 1770 */ 167, 400, 163, 96, 403, 404, 405, 406, 407, 408, + /* 1780 */ 163, 410, 169, 358, 95, 163, 227, 184, 417, 186, + /* 1790 */ 44, 366, 95, 422, 96, 44, 371, 95, 373, 358, + /* 1800 */ 95, 168, 168, 47, 95, 47, 44, 366, 96, 455, + /* 1810 */ 456, 95, 371, 96, 373, 96, 248, 3, 44, 96, + /* 1820 */ 35, 396, 35, 35, 35, 400, 35, 35, 403, 404, + /* 1830 */ 405, 406, 407, 408, 96, 410, 0, 396, 327, 44, + /* 1840 */ 47, 400, 0, 47, 403, 404, 405, 406, 407, 408, + /* 1850 */ 0, 410, 95, 0, 327, 96, 96, 242, 95, 95, + /* 1860 */ 248, 95, 39, 95, 47, 44, 105, 229, 2, 358, + /* 1870 */ 445, 22, 166, 205, 95, 95, 227, 366, 227, 164, + /* 1880 */ 47, 95, 371, 22, 373, 358, 96, 47, 447, 95, + /* 1890 */ 363, 207, 95, 366, 96, 96, 96, 327, 371, 95, + /* 1900 */ 373, 96, 35, 35, 95, 22, 96, 396, 106, 35, + /* 1910 */ 95, 400, 96, 35, 403, 404, 405, 406, 407, 408, + /* 1920 */ 95, 410, 96, 396, 35, 95, 35, 400, 358, 95, + /* 1930 */ 403, 404, 405, 406, 407, 408, 366, 410, 119, 96, + /* 1940 */ 95, 371, 107, 373, 44, 35, 119, 119, 119, 95, + /* 1950 */ 95, 22, 64, 63, 327, 35, 35, 35, 35, 35, + /* 1960 */ 35, 35, 35, 35, 92, 35, 396, 456, 70, 35, + /* 1970 */ 400, 35, 22, 403, 404, 405, 406, 407, 408, 44, + /* 1980 */ 410, 35, 412, 35, 35, 358, 35, 70, 35, 35, + /* 1990 */ 363, 35, 35, 366, 35, 22, 35, 0, 371, 35, + /* 2000 */ 373, 39, 0, 35, 39, 327, 48, 0, 48, 35, + /* 2010 */ 39, 48, 0, 35, 48, 39, 0, 35, 35, 0, + /* 2020 */ 22, 21, 21, 396, 20, 22, 327, 400, 22, 459, + /* 2030 */ 403, 404, 405, 406, 407, 408, 358, 410, 459, 459, + /* 2040 */ 459, 363, 459, 459, 366, 459, 459, 459, 327, 371, + /* 2050 */ 459, 373, 459, 459, 459, 459, 459, 358, 459, 459, + /* 2060 */ 459, 459, 459, 459, 459, 366, 459, 459, 459, 459, + /* 2070 */ 371, 459, 373, 459, 396, 459, 459, 459, 400, 358, + /* 2080 */ 459, 403, 404, 405, 406, 407, 408, 366, 410, 459, + /* 2090 */ 459, 459, 371, 459, 373, 396, 459, 459, 459, 400, + /* 2100 */ 459, 459, 403, 404, 405, 406, 407, 408, 459, 410, + /* 2110 */ 459, 327, 459, 459, 459, 459, 459, 396, 459, 459, + /* 2120 */ 459, 400, 459, 459, 403, 404, 405, 406, 407, 408, + /* 2130 */ 459, 410, 459, 459, 459, 327, 459, 459, 459, 459, + /* 2140 */ 459, 459, 358, 459, 459, 459, 459, 459, 459, 459, + /* 2150 */ 366, 459, 459, 459, 459, 371, 459, 373, 459, 459, + /* 2160 */ 459, 459, 459, 459, 459, 459, 358, 459, 459, 459, + /* 2170 */ 459, 459, 459, 459, 366, 459, 459, 459, 459, 371, + /* 2180 */ 396, 373, 459, 459, 400, 459, 459, 403, 404, 405, + /* 2190 */ 406, 407, 408, 459, 410, 459, 459, 327, 459, 459, + /* 2200 */ 459, 459, 459, 459, 396, 459, 459, 459, 400, 459, + /* 2210 */ 459, 403, 404, 405, 406, 407, 408, 459, 410, 459, + /* 2220 */ 459, 327, 459, 459, 459, 459, 459, 459, 358, 459, + /* 2230 */ 459, 459, 459, 459, 459, 459, 366, 459, 459, 459, + /* 2240 */ 459, 371, 459, 373, 459, 459, 459, 459, 459, 459, + /* 2250 */ 459, 459, 358, 459, 459, 459, 459, 459, 459, 459, + /* 2260 */ 366, 459, 459, 459, 327, 371, 396, 373, 459, 459, + /* 2270 */ 400, 459, 459, 403, 404, 405, 406, 407, 408, 459, + /* 2280 */ 410, 459, 327, 459, 459, 459, 459, 459, 459, 459, + /* 2290 */ 396, 459, 459, 459, 400, 358, 459, 403, 404, 405, + /* 2300 */ 406, 407, 408, 366, 410, 459, 459, 459, 371, 459, + /* 2310 */ 373, 459, 459, 358, 459, 459, 459, 459, 459, 459, + /* 2320 */ 459, 366, 459, 459, 459, 459, 371, 459, 373, 459, + /* 2330 */ 459, 459, 459, 396, 459, 459, 459, 400, 459, 459, + /* 2340 */ 403, 404, 405, 406, 407, 408, 459, 410, 459, 459, + /* 2350 */ 459, 396, 459, 459, 327, 400, 459, 459, 403, 404, + /* 2360 */ 405, 406, 407, 408, 459, 410, 459, 459, 459, 459, + /* 2370 */ 459, 459, 327, 459, 459, 459, 459, 459, 459, 459, + /* 2380 */ 459, 459, 459, 459, 459, 358, 459, 459, 459, 459, + /* 2390 */ 459, 459, 459, 366, 459, 459, 459, 459, 371, 459, + /* 2400 */ 373, 459, 459, 358, 459, 459, 459, 459, 459, 459, + /* 2410 */ 459, 366, 459, 459, 459, 327, 371, 459, 373, 459, + /* 2420 */ 459, 459, 459, 396, 459, 459, 459, 400, 459, 459, + /* 2430 */ 403, 404, 405, 406, 407, 408, 459, 410, 459, 327, + /* 2440 */ 459, 396, 459, 459, 459, 400, 358, 459, 403, 404, + /* 2450 */ 405, 406, 407, 408, 366, 410, 459, 459, 459, 371, + /* 2460 */ 459, 373, 459, 459, 459, 459, 459, 459, 459, 459, + /* 2470 */ 358, 459, 459, 459, 459, 459, 459, 459, 366, 459, + /* 2480 */ 459, 459, 459, 371, 396, 373, 459, 459, 400, 459, + /* 2490 */ 459, 403, 404, 405, 406, 407, 408, 459, 410, 459, + /* 2500 */ 459, 459, 327, 459, 459, 459, 459, 459, 396, 459, + /* 2510 */ 459, 459, 400, 459, 459, 403, 404, 405, 406, 407, + /* 2520 */ 408, 459, 410, 459, 459, 459, 459, 459, 459, 459, + /* 2530 */ 459, 459, 459, 358, 459, 459, 459, 459, 459, 459, + /* 2540 */ 459, 366, 459, 459, 459, 459, 371, 459, 373, 459, + /* 2550 */ 459, 459, 459, 459, 459, 459, 327, 459, 459, 459, + /* 2560 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 2570 */ 459, 396, 459, 459, 459, 400, 459, 459, 403, 404, + /* 2580 */ 405, 406, 407, 408, 459, 410, 459, 358, 459, 459, + /* 2590 */ 459, 459, 459, 459, 459, 366, 459, 459, 459, 459, + /* 2600 */ 371, 459, 373, 459, 459, 459, 459, 459, 459, 459, + /* 2610 */ 327, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 2620 */ 459, 459, 459, 459, 459, 396, 327, 459, 459, 400, + /* 2630 */ 459, 459, 403, 404, 405, 406, 407, 408, 459, 410, + /* 2640 */ 459, 358, 459, 459, 459, 459, 459, 459, 459, 366, + /* 2650 */ 459, 459, 459, 459, 371, 459, 373, 358, 459, 459, + /* 2660 */ 459, 459, 459, 459, 459, 366, 459, 459, 459, 327, + /* 2670 */ 371, 459, 373, 459, 459, 459, 459, 459, 459, 396, + /* 2680 */ 459, 459, 459, 400, 459, 459, 403, 404, 405, 406, + /* 2690 */ 407, 408, 459, 410, 459, 396, 459, 459, 459, 400, + /* 2700 */ 358, 459, 403, 404, 405, 406, 407, 408, 366, 410, + /* 2710 */ 459, 459, 459, 371, 459, 373, 459, 459, 459, 459, + /* 2720 */ 459, 459, 459, 459, 459, 459, 327, 459, 459, 459, + /* 2730 */ 459, 459, 459, 459, 459, 459, 459, 459, 396, 459, + /* 2740 */ 459, 459, 400, 327, 459, 403, 404, 405, 406, 407, + /* 2750 */ 408, 459, 410, 459, 459, 459, 459, 358, 459, 459, + /* 2760 */ 459, 459, 459, 459, 459, 366, 459, 459, 459, 327, + /* 2770 */ 371, 459, 373, 459, 358, 459, 459, 459, 459, 459, + /* 2780 */ 459, 459, 366, 459, 459, 459, 459, 371, 459, 373, + /* 2790 */ 459, 459, 459, 459, 459, 396, 459, 459, 459, 400, + /* 2800 */ 358, 459, 403, 404, 405, 406, 407, 408, 366, 410, + /* 2810 */ 459, 459, 396, 371, 459, 373, 400, 459, 459, 403, + /* 2820 */ 404, 405, 406, 407, 408, 459, 410, 459, 459, 459, + /* 2830 */ 327, 459, 459, 459, 459, 459, 459, 459, 396, 459, + /* 2840 */ 459, 459, 400, 459, 459, 403, 404, 405, 406, 407, + /* 2850 */ 408, 459, 410, 459, 459, 459, 459, 459, 459, 459, + /* 2860 */ 459, 358, 459, 459, 459, 459, 459, 459, 459, 366, + /* 2870 */ 459, 459, 459, 459, 371, 459, 373, 459, 459, 459, + /* 2880 */ 459, 459, 459, 459, 327, 459, 459, 459, 459, 459, + /* 2890 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 396, + /* 2900 */ 459, 459, 459, 400, 459, 459, 403, 404, 405, 406, + /* 2910 */ 407, 408, 459, 410, 459, 358, 459, 459, 459, 459, + /* 2920 */ 459, 459, 459, 366, 459, 459, 459, 459, 371, 459, + /* 2930 */ 373, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 2940 */ 459, 459, 459, 459, 459, 459, 459, 459, 459, 459, + /* 2950 */ 459, 459, 459, 396, 459, 459, 459, 400, 459, 459, + /* 2960 */ 403, 404, 405, 406, 407, 408, 459, 410, }; -#define YY_SHIFT_COUNT (706) +#define YY_SHIFT_COUNT (713) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2151) +#define YY_SHIFT_MAX (2019) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 925, 0, 71, 0, 288, 288, 288, 288, 288, 288, - /* 10 */ 288, 288, 288, 359, 574, 574, 645, 574, 574, 574, + /* 10 */ 288, 288, 288, 288, 288, 359, 574, 574, 645, 574, /* 20 */ 574, 574, 574, 574, 574, 574, 574, 574, 574, 574, /* 30 */ 574, 574, 574, 574, 574, 574, 574, 574, 574, 574, - /* 40 */ 574, 574, 574, 574, 574, 574, 61, 315, 211, 516, - /* 50 */ 62, 356, 399, 356, 211, 211, 1054, 1054, 356, 1054, - /* 60 */ 1054, 104, 356, 12, 12, 512, 512, 158, 31, 187, - /* 70 */ 187, 12, 12, 12, 12, 12, 12, 12, 12, 12, - /* 80 */ 12, 76, 12, 12, 168, 12, 442, 12, 12, 536, - /* 90 */ 12, 12, 536, 12, 536, 536, 536, 12, 498, 783, - /* 100 */ 34, 34, 162, 502, 1297, 1297, 1297, 1297, 1297, 1297, - /* 110 */ 1297, 1297, 1297, 1297, 1297, 1297, 1297, 1297, 1297, 1297, - /* 120 */ 1297, 1297, 1297, 1298, 968, 158, 31, 661, 402, 217, - /* 130 */ 217, 217, 668, 312, 312, 402, 572, 572, 572, 508, - /* 140 */ 442, 3, 3, 393, 536, 536, 569, 569, 508, 576, - /* 150 */ 215, 215, 215, 215, 215, 215, 215, 571, 655, 522, - /* 160 */ 451, 867, 59, 327, 95, 591, 712, 573, 726, 479, - /* 170 */ 694, 733, 214, 964, 936, 214, 1059, 244, 864, 1036, - /* 180 */ 1262, 1145, 1283, 1307, 1283, 1187, 1330, 1330, 1283, 1187, - /* 190 */ 1187, 1266, 1330, 1330, 1330, 1361, 1361, 1365, 76, 442, - /* 200 */ 76, 1373, 1375, 76, 1373, 76, 76, 76, 1330, 76, - /* 210 */ 1357, 1357, 1361, 536, 536, 536, 536, 536, 536, 536, - /* 220 */ 536, 536, 536, 536, 1330, 1361, 569, 1245, 1365, 498, - /* 230 */ 1264, 442, 498, 1330, 1307, 1307, 569, 1202, 1205, 569, - /* 240 */ 1202, 1205, 569, 569, 536, 1199, 1317, 1202, 1240, 1242, - /* 250 */ 1258, 1036, 1241, 1243, 1247, 1270, 572, 1495, 1330, 1373, - /* 260 */ 498, 1205, 569, 569, 569, 569, 569, 1205, 569, 1367, - /* 270 */ 498, 508, 498, 572, 1445, 1448, 569, 576, 1330, 498, - /* 280 */ 1528, 1361, 2544, 2544, 2544, 2544, 2544, 2544, 2544, 826, - /* 290 */ 92, 742, 587, 418, 15, 735, 374, 832, 400, 941, - /* 300 */ 941, 544, 941, 941, 941, 941, 941, 941, 941, 987, - /* 310 */ 314, 51, 410, 73, 73, 167, 666, 180, 19, 292, - /* 320 */ 121, 99, 406, 121, 121, 121, 929, 829, 1071, 1066, - /* 330 */ 989, 1007, 933, 945, 1064, 1113, 1146, 1154, 1190, 986, - /* 340 */ 1134, 1181, 1078, 907, 323, 260, 1182, 1183, 1186, 1189, - /* 350 */ 1191, 981, 1193, 1040, 1084, 58, 1206, 1144, 1223, 1224, - /* 360 */ 1249, 1260, 1273, 1274, 1051, 1276, 1301, 1211, 1284, 1576, - /* 370 */ 1578, 1393, 1581, 1582, 1541, 1584, 1551, 1388, 1553, 1554, - /* 380 */ 1556, 1394, 1592, 1559, 1560, 1397, 1597, 1400, 1601, 1567, - /* 390 */ 1606, 1593, 1613, 1583, 1435, 1439, 1620, 1621, 1446, 1450, - /* 400 */ 1623, 1626, 1580, 1630, 1633, 1634, 1599, 1643, 1644, 1645, - /* 410 */ 1646, 1647, 1648, 1650, 1651, 1501, 1618, 1656, 1506, 1659, - /* 420 */ 1660, 1667, 1669, 1670, 1671, 1672, 1673, 1680, 1681, 1682, - /* 430 */ 1683, 1684, 1685, 1687, 1635, 1678, 1688, 1689, 1690, 1695, - /* 440 */ 1653, 1676, 1696, 1697, 1557, 1693, 1698, 1665, 1702, 1649, - /* 450 */ 1703, 1652, 1704, 1706, 1666, 1677, 1668, 1662, 1701, 1664, - /* 460 */ 1705, 1674, 1717, 1686, 1691, 1722, 1723, 1725, 1700, 1558, - /* 470 */ 1727, 1740, 1741, 1699, 1742, 1744, 1710, 1708, 1707, 1748, - /* 480 */ 1714, 1712, 1726, 1761, 1729, 1718, 1728, 1762, 1734, 1724, - /* 490 */ 1731, 1771, 1774, 1775, 1777, 1679, 1709, 1743, 1757, 1782, - /* 500 */ 1750, 1751, 1765, 1753, 1755, 1747, 1749, 1760, 1763, 1770, - /* 510 */ 1764, 1796, 1778, 1803, 1783, 1758, 1807, 1787, 1786, 1822, - /* 520 */ 1790, 1828, 1794, 1830, 1811, 1823, 1745, 1752, 1846, 1713, - /* 530 */ 1815, 1851, 1711, 1833, 1730, 1720, 1852, 1864, 1732, 1736, - /* 540 */ 1863, 1821, 1624, 1772, 1776, 1793, 1789, 1827, 1800, 1795, - /* 550 */ 1812, 1813, 1801, 1866, 1860, 1865, 1818, 1867, 1694, 1831, - /* 560 */ 1832, 1913, 1875, 1721, 1895, 1897, 1899, 1900, 1901, 1902, - /* 570 */ 1841, 1842, 1893, 1715, 1903, 1896, 1944, 1945, 1948, 1853, - /* 580 */ 1854, 1855, 1857, 1915, 1920, 1766, 1962, 1877, 1805, 1878, - /* 590 */ 1975, 1937, 1814, 1881, 1873, 1662, 1934, 1938, 1756, 1768, - /* 600 */ 1773, 1984, 1967, 1785, 1906, 1904, 1907, 1908, 1910, 1911, - /* 610 */ 1957, 1916, 1917, 1960, 1912, 1993, 1804, 1929, 1919, 1926, - /* 620 */ 1992, 1994, 1932, 1935, 1996, 1939, 1936, 1999, 1940, 1942, - /* 630 */ 2002, 1946, 1943, 2006, 1950, 1924, 1928, 1930, 1931, 2027, - /* 640 */ 1949, 1956, 1958, 2018, 1965, 2012, 2012, 2036, 2000, 2003, - /* 650 */ 2028, 2030, 2032, 2034, 2035, 2038, 2039, 2046, 2049, 2051, - /* 660 */ 2001, 1995, 2043, 2056, 2065, 2079, 2067, 2082, 2071, 2072, - /* 670 */ 2074, 2040, 1747, 2076, 1749, 2077, 2078, 2080, 2081, 2086, - /* 680 */ 2083, 2117, 2084, 2075, 2085, 2120, 2090, 2096, 2091, 2121, - /* 690 */ 2098, 2097, 2092, 2134, 2106, 2099, 2107, 2148, 2114, 2115, - /* 700 */ 2151, 2131, 2133, 2145, 2146, 2137, 2135, + /* 40 */ 574, 574, 574, 574, 574, 574, 574, 574, 292, 437, + /* 50 */ 12, 294, 287, 151, 315, 151, 12, 12, 1207, 1207, + /* 60 */ 151, 1207, 1207, 78, 151, 19, 19, 198, 198, 66, + /* 70 */ 211, 98, 98, 19, 19, 19, 19, 19, 19, 19, + /* 80 */ 19, 19, 19, 112, 19, 19, 108, 19, 522, 19, + /* 90 */ 19, 536, 19, 19, 536, 19, 536, 536, 536, 19, + /* 100 */ 676, 783, 34, 34, 372, 103, 418, 418, 418, 418, + /* 110 */ 418, 418, 418, 418, 418, 418, 418, 418, 418, 418, + /* 120 */ 418, 418, 418, 418, 418, 213, 295, 66, 211, 62, + /* 130 */ 651, 1, 1, 1, 546, 79, 79, 651, 681, 681, + /* 140 */ 681, 606, 522, 129, 536, 706, 536, 706, 706, 606, + /* 150 */ 749, 216, 216, 216, 216, 216, 216, 216, 1603, 335, + /* 160 */ 36, 451, 867, 47, 410, 267, 46, 505, 227, 452, + /* 170 */ 792, 652, 639, 644, 596, 627, 644, 780, 892, 829, + /* 180 */ 1023, 1231, 1102, 1239, 1265, 1239, 1127, 1270, 1270, 1239, + /* 190 */ 1127, 1127, 1211, 1270, 1270, 1270, 1293, 1293, 1294, 112, + /* 200 */ 522, 112, 1306, 1309, 112, 1306, 112, 112, 112, 1270, + /* 210 */ 112, 1284, 1284, 1293, 536, 536, 536, 536, 536, 536, + /* 220 */ 536, 536, 536, 536, 536, 1270, 1293, 706, 706, 1153, + /* 230 */ 1294, 676, 1190, 522, 676, 1270, 1265, 1265, 706, 1128, + /* 240 */ 1132, 706, 1128, 1132, 706, 706, 536, 1121, 1215, 1128, + /* 250 */ 1134, 1136, 1157, 1023, 1138, 1143, 1146, 1172, 681, 1398, + /* 260 */ 1270, 1306, 676, 1132, 706, 706, 706, 706, 706, 1132, + /* 270 */ 706, 1286, 676, 606, 676, 681, 1365, 1367, 706, 749, + /* 280 */ 1270, 676, 1430, 1293, 2968, 2968, 2968, 2968, 2968, 2968, + /* 290 */ 2968, 2968, 2968, 826, 380, 455, 656, 514, 15, 967, + /* 300 */ 642, 59, 832, 938, 842, 1164, 1164, 1164, 1164, 1164, + /* 310 */ 1164, 1164, 1164, 1164, 963, 68, 184, 184, 365, 291, + /* 320 */ 181, 640, 226, 325, 325, 597, 669, 364, 597, 597, + /* 330 */ 597, 788, 1025, 980, 876, 833, 843, 786, 856, 857, + /* 340 */ 865, 1111, 1162, 1165, 289, 984, 985, 943, 951, 900, + /* 350 */ 851, 1070, 1131, 1159, 1160, 1161, 1130, 1166, 886, 1026, + /* 360 */ 290, 1167, 1151, 1171, 1174, 1177, 1179, 1182, 1186, 905, + /* 370 */ 1212, 1224, 1204, 766, 1485, 1486, 1300, 1488, 1489, 1448, + /* 380 */ 1491, 1457, 1295, 1458, 1460, 1461, 1298, 1498, 1464, 1465, + /* 390 */ 1302, 1502, 1304, 1504, 1471, 1507, 1499, 1508, 1476, 1338, + /* 400 */ 1344, 1532, 1533, 1351, 1353, 1529, 1534, 1490, 1535, 1536, + /* 410 */ 1538, 1497, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, + /* 420 */ 1400, 1514, 1554, 1412, 1564, 1565, 1567, 1575, 1576, 1577, + /* 430 */ 1578, 1579, 1580, 1582, 1583, 1584, 1585, 1586, 1587, 1528, + /* 440 */ 1571, 1572, 1588, 1590, 1592, 1589, 1593, 1597, 1604, 1472, + /* 450 */ 1606, 1608, 1574, 1610, 1555, 1614, 1557, 1618, 1620, 1595, + /* 460 */ 1600, 1581, 1594, 1607, 1596, 1626, 1598, 1624, 1602, 1605, + /* 470 */ 1646, 1647, 1648, 1611, 1478, 1652, 1653, 1654, 1591, 1656, + /* 480 */ 1662, 1628, 1616, 1629, 1665, 1631, 1619, 1632, 1670, 1637, + /* 490 */ 1625, 1635, 1675, 1641, 1630, 1640, 1680, 1683, 1685, 1686, + /* 500 */ 1599, 1621, 1655, 1666, 1687, 1657, 1658, 1659, 1660, 1645, + /* 510 */ 1661, 1664, 1667, 1669, 1671, 1696, 1679, 1704, 1688, 1663, + /* 520 */ 1708, 1700, 1674, 1724, 1691, 1728, 1695, 1731, 1710, 1714, + /* 530 */ 1701, 1703, 1548, 1639, 1644, 1741, 1609, 1711, 1743, 1562, + /* 540 */ 1725, 1617, 1601, 1748, 1749, 1622, 1613, 1750, 1707, 1515, + /* 550 */ 1689, 1673, 1697, 1633, 1559, 1634, 1513, 1677, 1746, 1698, + /* 560 */ 1702, 1705, 1709, 1712, 1751, 1756, 1758, 1716, 1762, 1568, + /* 570 */ 1717, 1719, 1814, 1774, 1612, 1785, 1787, 1788, 1789, 1791, + /* 580 */ 1792, 1723, 1738, 1793, 1615, 1795, 1796, 1836, 1842, 1850, + /* 590 */ 1757, 1759, 1760, 1763, 1764, 1706, 1766, 1853, 1823, 1715, + /* 600 */ 1768, 1761, 1594, 1817, 1821, 1649, 1638, 1651, 1866, 1849, + /* 610 */ 1668, 1779, 1790, 1780, 1798, 1786, 1799, 1833, 1794, 1797, + /* 620 */ 1840, 1800, 1861, 1684, 1804, 1802, 1805, 1867, 1868, 1809, + /* 630 */ 1810, 1874, 1815, 1816, 1878, 1825, 1826, 1889, 1830, 1843, + /* 640 */ 1891, 1834, 1819, 1827, 1828, 1829, 1883, 1835, 1845, 1900, + /* 650 */ 1854, 1910, 1855, 1900, 1900, 1929, 1888, 1890, 1920, 1921, + /* 660 */ 1922, 1923, 1924, 1925, 1926, 1927, 1928, 1930, 1898, 1872, + /* 670 */ 1935, 1934, 1936, 1946, 1950, 1948, 1949, 1951, 1917, 1645, + /* 680 */ 1953, 1661, 1954, 1956, 1957, 1959, 1973, 1961, 1997, 1964, + /* 690 */ 1958, 1962, 2002, 1968, 1960, 1965, 2007, 1974, 1963, 1971, + /* 700 */ 2012, 1978, 1966, 1976, 2016, 1982, 1983, 2019, 1998, 2000, + /* 710 */ 2003, 2006, 2001, 2004, }; -#define YY_REDUCE_COUNT (288) -#define YY_REDUCE_MIN (-425) -#define YY_REDUCE_MAX (2136) +#define YY_REDUCE_COUNT (292) +#define YY_REDUCE_MIN (-381) +#define YY_REDUCE_MAX (2557) static const short yy_reduce_ofst[] = { - /* 0 */ -76, -288, 286, 597, 650, 737, 780, 851, 894, 937, - /* 10 */ 961, 1019, 1062, 1090, -2, 83, 1149, 1207, 1235, 1261, - /* 20 */ 1331, 1350, 1413, 1434, 1456, 1477, 1499, 1520, 1563, 1591, - /* 30 */ 1617, 1675, 1692, 1735, 1759, 1802, 1819, 1845, 1862, 1905, - /* 40 */ 1927, 1989, 2011, 2073, 2093, 2136, -253, -196, 290, -385, - /* 50 */ -374, -323, 584, 778, 395, 659, -212, 764, 28, 408, - /* 60 */ 681, -425, -352, 72, 270, -324, -313, -339, -215, 23, - /* 70 */ 64, 267, 285, 331, 613, 615, 626, 649, 695, 700, - /* 80 */ 728, 298, 660, 748, 22, 790, -326, 833, 842, -345, - /* 90 */ 854, 857, -260, 869, 287, -190, 354, 710, 7, -189, - /* 100 */ -407, -407, -305, -233, -125, -66, 151, 193, 194, 221, - /* 110 */ 294, 340, 357, 407, 413, 484, 504, 506, 518, 519, - /* 120 */ 596, 651, 656, -56, -257, 283, -301, -185, -38, -257, - /* 130 */ 11, 412, 111, 401, 533, 18, -172, 234, 446, 411, - /* 140 */ 78, 586, 595, 425, -291, 468, 520, 628, 680, 727, - /* 150 */ 320, 328, 363, 493, 652, 684, 715, 321, 673, 765, - /* 160 */ 731, 698, 670, 825, 756, 840, 840, 868, 860, 908, - /* 170 */ 874, 865, 814, 814, 798, 814, 827, 871, 840, 914, - /* 180 */ 921, 935, 951, 950, 970, 975, 1020, 1021, 984, 996, - /* 190 */ 999, 1033, 1044, 1045, 1046, 1057, 1058, 997, 1052, 1022, - /* 200 */ 1053, 1061, 1010, 1060, 1068, 1065, 1067, 1069, 1072, 1070, - /* 210 */ 1075, 1076, 1087, 1063, 1074, 1079, 1080, 1081, 1085, 1086, - /* 220 */ 1088, 1089, 1091, 1092, 1095, 1105, 1082, 1050, 1077, 1112, - /* 230 */ 1094, 1100, 1115, 1120, 1083, 1093, 1104, 1038, 1097, 1107, - /* 240 */ 1049, 1101, 1118, 1121, 840, 1096, 1073, 1098, 1102, 1099, - /* 250 */ 1103, 1111, 1055, 1106, 1109, 814, 1151, 1117, 1188, 1185, - /* 260 */ 1192, 1140, 1155, 1156, 1157, 1158, 1159, 1141, 1161, 1160, - /* 270 */ 1194, 1196, 1204, 1175, 1126, 1201, 1177, 1203, 1226, 1225, - /* 280 */ 1238, 1246, 1178, 1195, 1218, 1220, 1222, 1236, 1252, + /* 0 */ -318, -250, 233, 324, 630, 650, 742, 785, 828, 895, + /* 10 */ 949, 1035, 1109, 1152, 1195, 1223, 1311, 1354, 1371, 1425, + /* 20 */ 1441, 1511, 1527, 1570, 1627, 1678, 1699, 1721, 1784, 1808, + /* 30 */ 1870, 1894, 1937, 1955, 2027, 2045, 2088, 2112, 2175, 2229, + /* 40 */ 2283, 2299, 2342, 2399, 2416, 2442, 2503, 2557, -191, 104, + /* 50 */ -18, -7, 168, 239, 269, 558, -278, 658, -345, -320, + /* 60 */ -270, -273, -170, -79, -10, 296, 328, -331, -326, -342, + /* 70 */ -328, -236, -193, 253, 254, 376, 398, 408, 409, 469, + /* 80 */ 643, 678, 682, -330, 683, 729, 185, 768, 41, 770, + /* 90 */ 775, -349, 795, 804, -280, 805, -245, 260, -113, 847, + /* 100 */ 16, -283, 130, 130, -309, -202, 163, 337, 389, 392, + /* 110 */ 434, 481, 501, 502, 503, 633, 634, 670, 671, 672, + /* 120 */ 695, 747, 750, 789, 841, -281, 87, 10, 220, 84, + /* 130 */ 283, 87, 245, 251, 344, -225, -38, 355, -313, 319, + /* 140 */ 360, 397, 45, 297, 196, 384, -110, 441, 443, 466, + /* 150 */ 475, -355, 519, 615, 720, 727, 737, 745, -381, 637, + /* 160 */ 759, 726, 665, 663, 846, 757, 883, 883, 872, 863, + /* 170 */ 880, 878, 870, 821, 821, 808, 821, 831, 830, 883, + /* 180 */ 873, 875, 894, 906, 907, 908, 913, 957, 958, 916, + /* 190 */ 920, 921, 965, 971, 974, 976, 989, 990, 928, 982, + /* 200 */ 952, 986, 992, 942, 991, 995, 993, 994, 996, 998, + /* 210 */ 997, 1001, 1002, 1012, 988, 1000, 1003, 1005, 1007, 1009, + /* 220 */ 1010, 1011, 1014, 1015, 1016, 1008, 1013, 973, 977, 955, + /* 230 */ 983, 1021, 959, 1004, 1039, 1044, 999, 1017, 1019, 944, + /* 240 */ 1018, 1020, 945, 1027, 1024, 1028, 883, 946, 953, 961, + /* 250 */ 954, 966, 969, 1022, 956, 964, 968, 821, 1051, 1030, + /* 260 */ 1086, 1087, 1083, 1049, 1063, 1066, 1073, 1075, 1076, 1064, + /* 270 */ 1078, 1071, 1115, 1101, 1119, 1093, 1041, 1103, 1092, 1116, + /* 280 */ 1133, 1135, 1139, 1141, 1081, 1079, 1091, 1096, 1120, 1125, + /* 290 */ 1126, 1144, 1158, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 10 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 20 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 30 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 40 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 50 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 60 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1843, 1587, 1587, - /* 70 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 80 */ 1587, 1665, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 90 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1663, 1836, - /* 100 */ 2032, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 110 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 120 */ 1587, 1587, 1587, 1587, 2044, 1587, 1587, 1665, 1587, 2044, - /* 130 */ 2044, 2044, 1663, 2004, 2004, 1587, 1587, 1587, 1587, 1774, - /* 140 */ 1587, 1885, 1885, 1587, 1587, 1587, 1587, 1587, 1774, 1587, - /* 150 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1879, 1587, 1587, - /* 160 */ 2069, 2122, 1587, 1587, 2072, 1587, 1587, 1587, 1848, 1587, - /* 170 */ 1727, 2059, 2036, 2050, 2106, 2037, 2034, 2053, 1587, 2063, - /* 180 */ 1587, 1872, 1841, 1587, 1841, 1838, 1587, 1587, 1841, 1838, - /* 190 */ 1838, 1718, 1587, 1587, 1587, 1587, 1587, 1587, 1665, 1587, - /* 200 */ 1665, 1587, 1587, 1665, 1587, 1665, 1665, 1665, 1587, 1665, - /* 210 */ 1644, 1644, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 220 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1892, 1587, 1663, - /* 230 */ 1881, 1587, 1663, 1587, 1587, 1587, 1587, 2079, 2077, 1587, - /* 240 */ 2079, 2077, 1587, 1587, 1587, 2091, 2087, 2079, 2095, 2093, - /* 250 */ 2065, 2063, 2125, 2112, 2108, 2050, 1587, 1587, 1587, 1587, - /* 260 */ 1663, 2077, 1587, 1587, 1587, 1587, 1587, 2077, 1587, 1587, - /* 270 */ 1663, 1587, 1663, 1587, 1587, 1743, 1587, 1587, 1587, 1663, - /* 280 */ 1619, 1587, 1874, 1885, 1777, 1777, 1777, 1666, 1592, 1587, - /* 290 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 2090, - /* 300 */ 2089, 1587, 1960, 1587, 2008, 2007, 2006, 1997, 1959, 1587, - /* 310 */ 1739, 1587, 1587, 1958, 1957, 1587, 1587, 1587, 1587, 1587, - /* 320 */ 1951, 1587, 1587, 1952, 1950, 1949, 1587, 1587, 1587, 1587, - /* 330 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 340 */ 1587, 1587, 1587, 2109, 2113, 1587, 1587, 1587, 1587, 1587, - /* 350 */ 1587, 2033, 1587, 1587, 1587, 1587, 1587, 1934, 1587, 1587, - /* 360 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 370 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 380 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 390 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 400 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 410 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 420 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 430 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 440 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 450 */ 1587, 1587, 1587, 1587, 1587, 1587, 1624, 1939, 1587, 1587, - /* 460 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 470 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 480 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 490 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 500 */ 1587, 1587, 1587, 1587, 1587, 1705, 1704, 1587, 1587, 1587, - /* 510 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 520 */ 1587, 1587, 1587, 1587, 1587, 1587, 1942, 1587, 1587, 1587, - /* 530 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 540 */ 2105, 2066, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 550 */ 1587, 1587, 1587, 1587, 1587, 1934, 1587, 2088, 1587, 1587, - /* 560 */ 2103, 1587, 2107, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 570 */ 2043, 2039, 1587, 1587, 2035, 1587, 1587, 1587, 1587, 1587, - /* 580 */ 1587, 1587, 1587, 1587, 1587, 1587, 1889, 1587, 1587, 1587, - /* 590 */ 1587, 1587, 1587, 1587, 1587, 1933, 1587, 1994, 1587, 1587, - /* 600 */ 1587, 2028, 1587, 1587, 1979, 1587, 1587, 1587, 1587, 1587, - /* 610 */ 1587, 1587, 1587, 1587, 1942, 1587, 1945, 1587, 1587, 1587, - /* 620 */ 1587, 1587, 1771, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 630 */ 1587, 1587, 1587, 1587, 1587, 1756, 1754, 1753, 1752, 1587, - /* 640 */ 1749, 1587, 1587, 1587, 1587, 1780, 1779, 1587, 1587, 1587, - /* 650 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 660 */ 1587, 1587, 1685, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 670 */ 1587, 1587, 1676, 1587, 1675, 1587, 1587, 1587, 1587, 1587, - /* 680 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 690 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, 1587, - /* 700 */ 1587, 1587, 1587, 1587, 1587, 1587, 1587, + /* 0 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 10 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 20 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 30 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 40 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 50 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 60 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1849, + /* 70 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 80 */ 1594, 1594, 1594, 1672, 1594, 1594, 1594, 1594, 1594, 1594, + /* 90 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 100 */ 1670, 1842, 2039, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 110 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 120 */ 1594, 1594, 1594, 1594, 1594, 1594, 2051, 1594, 1594, 1672, + /* 130 */ 1594, 2051, 2051, 2051, 1670, 2011, 2011, 1594, 1594, 1594, + /* 140 */ 1594, 1779, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1779, + /* 150 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1886, 1594, + /* 160 */ 1594, 2076, 2130, 1594, 1594, 2079, 1594, 1594, 1594, 1854, + /* 170 */ 1594, 1732, 2066, 2043, 2057, 2114, 2044, 2041, 2060, 1594, + /* 180 */ 2070, 1594, 1879, 1847, 1594, 1847, 1844, 1594, 1594, 1847, + /* 190 */ 1844, 1844, 1723, 1594, 1594, 1594, 1594, 1594, 1594, 1672, + /* 200 */ 1594, 1672, 1594, 1594, 1672, 1594, 1672, 1672, 1672, 1594, + /* 210 */ 1672, 1651, 1651, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 220 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1899, + /* 230 */ 1594, 1670, 1888, 1594, 1670, 1594, 1594, 1594, 1594, 2087, + /* 240 */ 2085, 1594, 2087, 2085, 1594, 1594, 1594, 2099, 2095, 2087, + /* 250 */ 2103, 2101, 2072, 2070, 2133, 2120, 2116, 2057, 1594, 1594, + /* 260 */ 1594, 1594, 1670, 2085, 1594, 1594, 1594, 1594, 1594, 2085, + /* 270 */ 1594, 1594, 1670, 1594, 1670, 1594, 1594, 1748, 1594, 1594, + /* 280 */ 1594, 1670, 1626, 1594, 1881, 1892, 1864, 1864, 1782, 1782, + /* 290 */ 1782, 1673, 1599, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 300 */ 1594, 1594, 1594, 1594, 1594, 2098, 2097, 1967, 1594, 2015, + /* 310 */ 2014, 2013, 2004, 1966, 1744, 1594, 1965, 1964, 1594, 1594, + /* 320 */ 1594, 1594, 1594, 1860, 1859, 1958, 1594, 1594, 1959, 1957, + /* 330 */ 1956, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 340 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 2117, 2121, + /* 350 */ 1594, 1594, 1594, 1594, 1594, 1594, 2040, 1594, 1594, 1594, + /* 360 */ 1594, 1594, 1941, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 370 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 380 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 390 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 400 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 410 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 420 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 430 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 440 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 450 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 460 */ 1594, 1631, 1946, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 470 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 480 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 490 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 500 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1711, + /* 510 */ 1710, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 520 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 530 */ 1594, 1594, 1594, 1949, 1594, 1594, 1594, 1594, 1594, 1594, + /* 540 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 2113, 2073, 1594, + /* 550 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 560 */ 1594, 1594, 1594, 1594, 1594, 1594, 1941, 1594, 2096, 1594, + /* 570 */ 1594, 2111, 1594, 2115, 1594, 1594, 1594, 1594, 1594, 1594, + /* 580 */ 1594, 2050, 2046, 1594, 1594, 2042, 1594, 1594, 1594, 1594, + /* 590 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 600 */ 1594, 1594, 1940, 1594, 2001, 1594, 1594, 1594, 2035, 1594, + /* 610 */ 1594, 1986, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 620 */ 1594, 1949, 1594, 1952, 1594, 1594, 1594, 1594, 1594, 1776, + /* 630 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 640 */ 1594, 1594, 1761, 1759, 1758, 1757, 1594, 1754, 1594, 1789, + /* 650 */ 1594, 1594, 1594, 1785, 1784, 1594, 1594, 1594, 1594, 1594, + /* 660 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 670 */ 1691, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1683, + /* 680 */ 1594, 1682, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 690 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 700 */ 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, 1594, + /* 710 */ 1594, 1594, 1594, 1594, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1011,7 +1098,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* TSDB_PAGESIZE => nothing */ 0, /* PRECISION => nothing */ 0, /* REPLICA => nothing */ - 0, /* STRICT => nothing */ 0, /* VGROUPS => nothing */ 0, /* SINGLE_STABLE => nothing */ 0, /* RETENTIONS => nothing */ @@ -1064,6 +1150,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* ROLLUP => nothing */ 0, /* TTL => nothing */ 0, /* SMA => nothing */ + 0, /* DELETE_MARK => nothing */ 0, /* FIRST => nothing */ 0, /* LAST => nothing */ 0, /* SHOW => nothing */ @@ -1159,7 +1246,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* COUNT => nothing */ 0, /* LAST_ROW => nothing */ 0, /* CASE => nothing */ - 268, /* END => ABORT */ + 270, /* END => ABORT */ 0, /* WHEN => nothing */ 0, /* THEN => nothing */ 0, /* ELSE => nothing */ @@ -1183,6 +1270,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* BY => nothing */ 0, /* SESSION => nothing */ 0, /* STATE_WINDOW => nothing */ + 0, /* EVENT_WINDOW => nothing */ + 0, /* START => nothing */ 0, /* SLIDING => nothing */ 0, /* FILL => nothing */ 0, /* VALUE => nothing */ @@ -1201,58 +1290,59 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 268, /* AFTER => ABORT */ - 268, /* ATTACH => ABORT */ - 268, /* BEFORE => ABORT */ - 268, /* BEGIN => ABORT */ - 268, /* BITAND => ABORT */ - 268, /* BITNOT => ABORT */ - 268, /* BITOR => ABORT */ - 268, /* BLOCKS => ABORT */ - 268, /* CHANGE => ABORT */ - 268, /* COMMA => ABORT */ - 268, /* COMPACT => ABORT */ - 268, /* CONCAT => ABORT */ - 268, /* CONFLICT => ABORT */ - 268, /* COPY => ABORT */ - 268, /* DEFERRED => ABORT */ - 268, /* DELIMITERS => ABORT */ - 268, /* DETACH => ABORT */ - 268, /* DIVIDE => ABORT */ - 268, /* DOT => ABORT */ - 268, /* EACH => ABORT */ - 268, /* FAIL => ABORT */ - 268, /* FILE => ABORT */ - 268, /* FOR => ABORT */ - 268, /* GLOB => ABORT */ - 268, /* ID => ABORT */ - 268, /* IMMEDIATE => ABORT */ - 268, /* IMPORT => ABORT */ - 268, /* INITIALLY => ABORT */ - 268, /* INSTEAD => ABORT */ - 268, /* ISNULL => ABORT */ - 268, /* KEY => ABORT */ - 268, /* MODULES => ABORT */ - 268, /* NK_BITNOT => ABORT */ - 268, /* NK_SEMI => ABORT */ - 268, /* NOTNULL => ABORT */ - 268, /* OF => ABORT */ - 268, /* PLUS => ABORT */ - 268, /* PRIVILEGE => ABORT */ - 268, /* RAISE => ABORT */ - 268, /* REPLACE => ABORT */ - 268, /* RESTRICT => ABORT */ - 268, /* ROW => ABORT */ - 268, /* SEMI => ABORT */ - 268, /* STAR => ABORT */ - 268, /* STATEMENT => ABORT */ - 268, /* STRING => ABORT */ - 268, /* TIMES => ABORT */ - 268, /* UPDATE => ABORT */ - 268, /* VALUES => ABORT */ - 268, /* VARIABLE => ABORT */ - 268, /* VIEW => ABORT */ - 268, /* WAL => ABORT */ + 270, /* AFTER => ABORT */ + 270, /* ATTACH => ABORT */ + 270, /* BEFORE => ABORT */ + 270, /* BEGIN => ABORT */ + 270, /* BITAND => ABORT */ + 270, /* BITNOT => ABORT */ + 270, /* BITOR => ABORT */ + 270, /* BLOCKS => ABORT */ + 270, /* CHANGE => ABORT */ + 270, /* COMMA => ABORT */ + 270, /* COMPACT => ABORT */ + 270, /* CONCAT => ABORT */ + 270, /* CONFLICT => ABORT */ + 270, /* COPY => ABORT */ + 270, /* DEFERRED => ABORT */ + 270, /* DELIMITERS => ABORT */ + 270, /* DETACH => ABORT */ + 270, /* DIVIDE => ABORT */ + 270, /* DOT => ABORT */ + 270, /* EACH => ABORT */ + 270, /* FAIL => ABORT */ + 270, /* FILE => ABORT */ + 270, /* FOR => ABORT */ + 270, /* GLOB => ABORT */ + 270, /* ID => ABORT */ + 270, /* IMMEDIATE => ABORT */ + 270, /* IMPORT => ABORT */ + 270, /* INITIALLY => ABORT */ + 270, /* INSTEAD => ABORT */ + 270, /* ISNULL => ABORT */ + 270, /* KEY => ABORT */ + 270, /* MODULES => ABORT */ + 270, /* NK_BITNOT => ABORT */ + 270, /* NK_SEMI => ABORT */ + 270, /* NOTNULL => ABORT */ + 270, /* OF => ABORT */ + 270, /* PLUS => ABORT */ + 270, /* PRIVILEGE => ABORT */ + 270, /* RAISE => ABORT */ + 270, /* REPLACE => ABORT */ + 270, /* RESTRICT => ABORT */ + 270, /* ROW => ABORT */ + 270, /* SEMI => ABORT */ + 270, /* STAR => ABORT */ + 270, /* STATEMENT => ABORT */ + 270, /* STRICT => ABORT */ + 270, /* STRING => ABORT */ + 270, /* TIMES => ABORT */ + 270, /* UPDATE => ABORT */ + 270, /* VALUES => ABORT */ + 270, /* VARIABLE => ABORT */ + 270, /* VIEW => ABORT */ + 270, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1419,59 +1509,59 @@ static const char *const yyTokenName[] = { /* 76 */ "TSDB_PAGESIZE", /* 77 */ "PRECISION", /* 78 */ "REPLICA", - /* 79 */ "STRICT", - /* 80 */ "VGROUPS", - /* 81 */ "SINGLE_STABLE", - /* 82 */ "RETENTIONS", - /* 83 */ "SCHEMALESS", - /* 84 */ "WAL_LEVEL", - /* 85 */ "WAL_FSYNC_PERIOD", - /* 86 */ "WAL_RETENTION_PERIOD", - /* 87 */ "WAL_RETENTION_SIZE", - /* 88 */ "WAL_ROLL_PERIOD", - /* 89 */ "WAL_SEGMENT_SIZE", - /* 90 */ "STT_TRIGGER", - /* 91 */ "TABLE_PREFIX", - /* 92 */ "TABLE_SUFFIX", - /* 93 */ "NK_COLON", - /* 94 */ "MAX_SPEED", - /* 95 */ "TABLE", - /* 96 */ "NK_LP", - /* 97 */ "NK_RP", - /* 98 */ "STABLE", - /* 99 */ "ADD", - /* 100 */ "COLUMN", - /* 101 */ "MODIFY", - /* 102 */ "RENAME", - /* 103 */ "TAG", - /* 104 */ "SET", - /* 105 */ "NK_EQ", - /* 106 */ "USING", - /* 107 */ "TAGS", - /* 108 */ "COMMENT", - /* 109 */ "BOOL", - /* 110 */ "TINYINT", - /* 111 */ "SMALLINT", - /* 112 */ "INT", - /* 113 */ "INTEGER", - /* 114 */ "BIGINT", - /* 115 */ "FLOAT", - /* 116 */ "DOUBLE", - /* 117 */ "BINARY", - /* 118 */ "TIMESTAMP", - /* 119 */ "NCHAR", - /* 120 */ "UNSIGNED", - /* 121 */ "JSON", - /* 122 */ "VARCHAR", - /* 123 */ "MEDIUMBLOB", - /* 124 */ "BLOB", - /* 125 */ "VARBINARY", - /* 126 */ "DECIMAL", - /* 127 */ "MAX_DELAY", - /* 128 */ "WATERMARK", - /* 129 */ "ROLLUP", - /* 130 */ "TTL", - /* 131 */ "SMA", + /* 79 */ "VGROUPS", + /* 80 */ "SINGLE_STABLE", + /* 81 */ "RETENTIONS", + /* 82 */ "SCHEMALESS", + /* 83 */ "WAL_LEVEL", + /* 84 */ "WAL_FSYNC_PERIOD", + /* 85 */ "WAL_RETENTION_PERIOD", + /* 86 */ "WAL_RETENTION_SIZE", + /* 87 */ "WAL_ROLL_PERIOD", + /* 88 */ "WAL_SEGMENT_SIZE", + /* 89 */ "STT_TRIGGER", + /* 90 */ "TABLE_PREFIX", + /* 91 */ "TABLE_SUFFIX", + /* 92 */ "NK_COLON", + /* 93 */ "MAX_SPEED", + /* 94 */ "TABLE", + /* 95 */ "NK_LP", + /* 96 */ "NK_RP", + /* 97 */ "STABLE", + /* 98 */ "ADD", + /* 99 */ "COLUMN", + /* 100 */ "MODIFY", + /* 101 */ "RENAME", + /* 102 */ "TAG", + /* 103 */ "SET", + /* 104 */ "NK_EQ", + /* 105 */ "USING", + /* 106 */ "TAGS", + /* 107 */ "COMMENT", + /* 108 */ "BOOL", + /* 109 */ "TINYINT", + /* 110 */ "SMALLINT", + /* 111 */ "INT", + /* 112 */ "INTEGER", + /* 113 */ "BIGINT", + /* 114 */ "FLOAT", + /* 115 */ "DOUBLE", + /* 116 */ "BINARY", + /* 117 */ "TIMESTAMP", + /* 118 */ "NCHAR", + /* 119 */ "UNSIGNED", + /* 120 */ "JSON", + /* 121 */ "VARCHAR", + /* 122 */ "MEDIUMBLOB", + /* 123 */ "BLOB", + /* 124 */ "VARBINARY", + /* 125 */ "DECIMAL", + /* 126 */ "MAX_DELAY", + /* 127 */ "WATERMARK", + /* 128 */ "ROLLUP", + /* 129 */ "TTL", + /* 130 */ "SMA", + /* 131 */ "DELETE_MARK", /* 132 */ "FIRST", /* 133 */ "LAST", /* 134 */ "SHOW", @@ -1591,211 +1681,214 @@ static const char *const yyTokenName[] = { /* 248 */ "BY", /* 249 */ "SESSION", /* 250 */ "STATE_WINDOW", - /* 251 */ "SLIDING", - /* 252 */ "FILL", - /* 253 */ "VALUE", - /* 254 */ "NONE", - /* 255 */ "PREV", - /* 256 */ "LINEAR", - /* 257 */ "NEXT", - /* 258 */ "HAVING", - /* 259 */ "RANGE", - /* 260 */ "EVERY", - /* 261 */ "ORDER", - /* 262 */ "SLIMIT", - /* 263 */ "SOFFSET", - /* 264 */ "LIMIT", - /* 265 */ "OFFSET", - /* 266 */ "ASC", - /* 267 */ "NULLS", - /* 268 */ "ABORT", - /* 269 */ "AFTER", - /* 270 */ "ATTACH", - /* 271 */ "BEFORE", - /* 272 */ "BEGIN", - /* 273 */ "BITAND", - /* 274 */ "BITNOT", - /* 275 */ "BITOR", - /* 276 */ "BLOCKS", - /* 277 */ "CHANGE", - /* 278 */ "COMMA", - /* 279 */ "COMPACT", - /* 280 */ "CONCAT", - /* 281 */ "CONFLICT", - /* 282 */ "COPY", - /* 283 */ "DEFERRED", - /* 284 */ "DELIMITERS", - /* 285 */ "DETACH", - /* 286 */ "DIVIDE", - /* 287 */ "DOT", - /* 288 */ "EACH", - /* 289 */ "FAIL", - /* 290 */ "FILE", - /* 291 */ "FOR", - /* 292 */ "GLOB", - /* 293 */ "ID", - /* 294 */ "IMMEDIATE", - /* 295 */ "IMPORT", - /* 296 */ "INITIALLY", - /* 297 */ "INSTEAD", - /* 298 */ "ISNULL", - /* 299 */ "KEY", - /* 300 */ "MODULES", - /* 301 */ "NK_BITNOT", - /* 302 */ "NK_SEMI", - /* 303 */ "NOTNULL", - /* 304 */ "OF", - /* 305 */ "PLUS", - /* 306 */ "PRIVILEGE", - /* 307 */ "RAISE", - /* 308 */ "REPLACE", - /* 309 */ "RESTRICT", - /* 310 */ "ROW", - /* 311 */ "SEMI", - /* 312 */ "STAR", - /* 313 */ "STATEMENT", - /* 314 */ "STRING", - /* 315 */ "TIMES", - /* 316 */ "UPDATE", - /* 317 */ "VALUES", - /* 318 */ "VARIABLE", - /* 319 */ "VIEW", - /* 320 */ "WAL", - /* 321 */ "cmd", - /* 322 */ "account_options", - /* 323 */ "alter_account_options", - /* 324 */ "literal", - /* 325 */ "alter_account_option", - /* 326 */ "user_name", - /* 327 */ "sysinfo_opt", - /* 328 */ "privileges", - /* 329 */ "priv_level", - /* 330 */ "priv_type_list", - /* 331 */ "priv_type", - /* 332 */ "db_name", - /* 333 */ "topic_name", - /* 334 */ "dnode_endpoint", - /* 335 */ "force_opt", - /* 336 */ "not_exists_opt", - /* 337 */ "db_options", - /* 338 */ "exists_opt", - /* 339 */ "alter_db_options", - /* 340 */ "speed_opt", - /* 341 */ "integer_list", - /* 342 */ "variable_list", - /* 343 */ "retention_list", - /* 344 */ "alter_db_option", - /* 345 */ "retention", - /* 346 */ "full_table_name", - /* 347 */ "column_def_list", - /* 348 */ "tags_def_opt", - /* 349 */ "table_options", - /* 350 */ "multi_create_clause", - /* 351 */ "tags_def", - /* 352 */ "multi_drop_clause", - /* 353 */ "alter_table_clause", - /* 354 */ "alter_table_options", - /* 355 */ "column_name", - /* 356 */ "type_name", - /* 357 */ "signed_literal", - /* 358 */ "create_subtable_clause", - /* 359 */ "specific_cols_opt", - /* 360 */ "expression_list", - /* 361 */ "drop_table_clause", - /* 362 */ "col_name_list", - /* 363 */ "table_name", - /* 364 */ "column_def", - /* 365 */ "duration_list", - /* 366 */ "rollup_func_list", - /* 367 */ "alter_table_option", - /* 368 */ "duration_literal", - /* 369 */ "rollup_func_name", - /* 370 */ "function_name", - /* 371 */ "col_name", - /* 372 */ "db_name_cond_opt", - /* 373 */ "like_pattern_opt", - /* 374 */ "table_name_cond", - /* 375 */ "from_db_opt", - /* 376 */ "tag_list_opt", - /* 377 */ "tag_item", - /* 378 */ "column_alias", - /* 379 */ "index_options", - /* 380 */ "func_list", - /* 381 */ "sliding_opt", - /* 382 */ "sma_stream_opt", - /* 383 */ "func", - /* 384 */ "stream_options", - /* 385 */ "query_or_subquery", - /* 386 */ "cgroup_name", - /* 387 */ "analyze_opt", - /* 388 */ "explain_options", - /* 389 */ "agg_func_opt", - /* 390 */ "bufsize_opt", - /* 391 */ "stream_name", - /* 392 */ "subtable_opt", - /* 393 */ "expression", - /* 394 */ "dnode_list", - /* 395 */ "where_clause_opt", - /* 396 */ "signed", - /* 397 */ "literal_func", - /* 398 */ "literal_list", - /* 399 */ "table_alias", - /* 400 */ "expr_or_subquery", - /* 401 */ "pseudo_column", - /* 402 */ "column_reference", - /* 403 */ "function_expression", - /* 404 */ "case_when_expression", - /* 405 */ "star_func", - /* 406 */ "star_func_para_list", - /* 407 */ "noarg_func", - /* 408 */ "other_para_list", - /* 409 */ "star_func_para", - /* 410 */ "when_then_list", - /* 411 */ "case_when_else_opt", - /* 412 */ "common_expression", - /* 413 */ "when_then_expr", - /* 414 */ "predicate", - /* 415 */ "compare_op", - /* 416 */ "in_op", - /* 417 */ "in_predicate_value", - /* 418 */ "boolean_value_expression", - /* 419 */ "boolean_primary", - /* 420 */ "from_clause_opt", - /* 421 */ "table_reference_list", - /* 422 */ "table_reference", - /* 423 */ "table_primary", - /* 424 */ "joined_table", - /* 425 */ "alias_opt", - /* 426 */ "subquery", - /* 427 */ "parenthesized_joined_table", - /* 428 */ "join_type", - /* 429 */ "search_condition", - /* 430 */ "query_specification", - /* 431 */ "set_quantifier_opt", - /* 432 */ "select_list", - /* 433 */ "partition_by_clause_opt", - /* 434 */ "range_opt", - /* 435 */ "every_opt", - /* 436 */ "fill_opt", - /* 437 */ "twindow_clause_opt", - /* 438 */ "group_by_clause_opt", - /* 439 */ "having_clause_opt", - /* 440 */ "select_item", - /* 441 */ "partition_list", - /* 442 */ "partition_item", - /* 443 */ "fill_mode", - /* 444 */ "group_by_list", - /* 445 */ "query_expression", - /* 446 */ "query_simple", - /* 447 */ "order_by_clause_opt", - /* 448 */ "slimit_clause_opt", - /* 449 */ "limit_clause_opt", - /* 450 */ "union_query_expression", - /* 451 */ "query_simple_or_subquery", - /* 452 */ "sort_specification_list", - /* 453 */ "sort_specification", - /* 454 */ "ordering_specification_opt", - /* 455 */ "null_ordering_opt", + /* 251 */ "EVENT_WINDOW", + /* 252 */ "START", + /* 253 */ "SLIDING", + /* 254 */ "FILL", + /* 255 */ "VALUE", + /* 256 */ "NONE", + /* 257 */ "PREV", + /* 258 */ "LINEAR", + /* 259 */ "NEXT", + /* 260 */ "HAVING", + /* 261 */ "RANGE", + /* 262 */ "EVERY", + /* 263 */ "ORDER", + /* 264 */ "SLIMIT", + /* 265 */ "SOFFSET", + /* 266 */ "LIMIT", + /* 267 */ "OFFSET", + /* 268 */ "ASC", + /* 269 */ "NULLS", + /* 270 */ "ABORT", + /* 271 */ "AFTER", + /* 272 */ "ATTACH", + /* 273 */ "BEFORE", + /* 274 */ "BEGIN", + /* 275 */ "BITAND", + /* 276 */ "BITNOT", + /* 277 */ "BITOR", + /* 278 */ "BLOCKS", + /* 279 */ "CHANGE", + /* 280 */ "COMMA", + /* 281 */ "COMPACT", + /* 282 */ "CONCAT", + /* 283 */ "CONFLICT", + /* 284 */ "COPY", + /* 285 */ "DEFERRED", + /* 286 */ "DELIMITERS", + /* 287 */ "DETACH", + /* 288 */ "DIVIDE", + /* 289 */ "DOT", + /* 290 */ "EACH", + /* 291 */ "FAIL", + /* 292 */ "FILE", + /* 293 */ "FOR", + /* 294 */ "GLOB", + /* 295 */ "ID", + /* 296 */ "IMMEDIATE", + /* 297 */ "IMPORT", + /* 298 */ "INITIALLY", + /* 299 */ "INSTEAD", + /* 300 */ "ISNULL", + /* 301 */ "KEY", + /* 302 */ "MODULES", + /* 303 */ "NK_BITNOT", + /* 304 */ "NK_SEMI", + /* 305 */ "NOTNULL", + /* 306 */ "OF", + /* 307 */ "PLUS", + /* 308 */ "PRIVILEGE", + /* 309 */ "RAISE", + /* 310 */ "REPLACE", + /* 311 */ "RESTRICT", + /* 312 */ "ROW", + /* 313 */ "SEMI", + /* 314 */ "STAR", + /* 315 */ "STATEMENT", + /* 316 */ "STRICT", + /* 317 */ "STRING", + /* 318 */ "TIMES", + /* 319 */ "UPDATE", + /* 320 */ "VALUES", + /* 321 */ "VARIABLE", + /* 322 */ "VIEW", + /* 323 */ "WAL", + /* 324 */ "cmd", + /* 325 */ "account_options", + /* 326 */ "alter_account_options", + /* 327 */ "literal", + /* 328 */ "alter_account_option", + /* 329 */ "user_name", + /* 330 */ "sysinfo_opt", + /* 331 */ "privileges", + /* 332 */ "priv_level", + /* 333 */ "priv_type_list", + /* 334 */ "priv_type", + /* 335 */ "db_name", + /* 336 */ "topic_name", + /* 337 */ "dnode_endpoint", + /* 338 */ "force_opt", + /* 339 */ "not_exists_opt", + /* 340 */ "db_options", + /* 341 */ "exists_opt", + /* 342 */ "alter_db_options", + /* 343 */ "speed_opt", + /* 344 */ "integer_list", + /* 345 */ "variable_list", + /* 346 */ "retention_list", + /* 347 */ "alter_db_option", + /* 348 */ "retention", + /* 349 */ "full_table_name", + /* 350 */ "column_def_list", + /* 351 */ "tags_def_opt", + /* 352 */ "table_options", + /* 353 */ "multi_create_clause", + /* 354 */ "tags_def", + /* 355 */ "multi_drop_clause", + /* 356 */ "alter_table_clause", + /* 357 */ "alter_table_options", + /* 358 */ "column_name", + /* 359 */ "type_name", + /* 360 */ "signed_literal", + /* 361 */ "create_subtable_clause", + /* 362 */ "specific_cols_opt", + /* 363 */ "expression_list", + /* 364 */ "drop_table_clause", + /* 365 */ "col_name_list", + /* 366 */ "table_name", + /* 367 */ "column_def", + /* 368 */ "duration_list", + /* 369 */ "rollup_func_list", + /* 370 */ "alter_table_option", + /* 371 */ "duration_literal", + /* 372 */ "rollup_func_name", + /* 373 */ "function_name", + /* 374 */ "col_name", + /* 375 */ "db_name_cond_opt", + /* 376 */ "like_pattern_opt", + /* 377 */ "table_name_cond", + /* 378 */ "from_db_opt", + /* 379 */ "tag_list_opt", + /* 380 */ "tag_item", + /* 381 */ "column_alias", + /* 382 */ "index_options", + /* 383 */ "func_list", + /* 384 */ "sliding_opt", + /* 385 */ "sma_stream_opt", + /* 386 */ "func", + /* 387 */ "query_or_subquery", + /* 388 */ "cgroup_name", + /* 389 */ "analyze_opt", + /* 390 */ "explain_options", + /* 391 */ "agg_func_opt", + /* 392 */ "bufsize_opt", + /* 393 */ "stream_name", + /* 394 */ "stream_options", + /* 395 */ "subtable_opt", + /* 396 */ "expression", + /* 397 */ "dnode_list", + /* 398 */ "where_clause_opt", + /* 399 */ "signed", + /* 400 */ "literal_func", + /* 401 */ "literal_list", + /* 402 */ "table_alias", + /* 403 */ "expr_or_subquery", + /* 404 */ "pseudo_column", + /* 405 */ "column_reference", + /* 406 */ "function_expression", + /* 407 */ "case_when_expression", + /* 408 */ "star_func", + /* 409 */ "star_func_para_list", + /* 410 */ "noarg_func", + /* 411 */ "other_para_list", + /* 412 */ "star_func_para", + /* 413 */ "when_then_list", + /* 414 */ "case_when_else_opt", + /* 415 */ "common_expression", + /* 416 */ "when_then_expr", + /* 417 */ "predicate", + /* 418 */ "compare_op", + /* 419 */ "in_op", + /* 420 */ "in_predicate_value", + /* 421 */ "boolean_value_expression", + /* 422 */ "boolean_primary", + /* 423 */ "from_clause_opt", + /* 424 */ "table_reference_list", + /* 425 */ "table_reference", + /* 426 */ "table_primary", + /* 427 */ "joined_table", + /* 428 */ "alias_opt", + /* 429 */ "subquery", + /* 430 */ "parenthesized_joined_table", + /* 431 */ "join_type", + /* 432 */ "search_condition", + /* 433 */ "query_specification", + /* 434 */ "set_quantifier_opt", + /* 435 */ "select_list", + /* 436 */ "partition_by_clause_opt", + /* 437 */ "range_opt", + /* 438 */ "every_opt", + /* 439 */ "fill_opt", + /* 440 */ "twindow_clause_opt", + /* 441 */ "group_by_clause_opt", + /* 442 */ "having_clause_opt", + /* 443 */ "select_item", + /* 444 */ "partition_list", + /* 445 */ "partition_item", + /* 446 */ "fill_mode", + /* 447 */ "group_by_list", + /* 448 */ "query_expression", + /* 449 */ "query_simple", + /* 450 */ "order_by_clause_opt", + /* 451 */ "slimit_clause_opt", + /* 452 */ "limit_clause_opt", + /* 453 */ "union_query_expression", + /* 454 */ "query_simple_or_subquery", + /* 455 */ "sort_specification_list", + /* 456 */ "sort_specification", + /* 457 */ "ordering_specification_opt", + /* 458 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1895,185 +1988,185 @@ static const char *const yyRuleName[] = { /* 89 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", /* 90 */ "db_options ::= db_options PRECISION NK_STRING", /* 91 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 92 */ "db_options ::= db_options STRICT NK_STRING", - /* 93 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 94 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 95 */ "db_options ::= db_options RETENTIONS retention_list", - /* 96 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 97 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 98 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 100 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 102 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 103 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 104 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 105 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 106 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 107 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", - /* 108 */ "alter_db_options ::= alter_db_option", - /* 109 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 110 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 111 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 112 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 113 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 114 */ "alter_db_option ::= KEEP integer_list", - /* 115 */ "alter_db_option ::= KEEP variable_list", - /* 116 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 117 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 118 */ "alter_db_option ::= STRICT NK_STRING", - /* 119 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 120 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", - /* 121 */ "integer_list ::= NK_INTEGER", - /* 122 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 123 */ "variable_list ::= NK_VARIABLE", - /* 124 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 125 */ "retention_list ::= retention", - /* 126 */ "retention_list ::= retention_list NK_COMMA retention", - /* 127 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 128 */ "speed_opt ::=", - /* 129 */ "speed_opt ::= MAX_SPEED NK_INTEGER", - /* 130 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 131 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 132 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 133 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 134 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 135 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 136 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 137 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 138 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 139 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 140 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 141 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 142 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 143 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 144 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 145 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 146 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 147 */ "multi_create_clause ::= create_subtable_clause", - /* 148 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 149 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 150 */ "multi_drop_clause ::= drop_table_clause", - /* 151 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 152 */ "drop_table_clause ::= exists_opt full_table_name", - /* 153 */ "specific_cols_opt ::=", - /* 154 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 155 */ "full_table_name ::= table_name", - /* 156 */ "full_table_name ::= db_name NK_DOT table_name", - /* 157 */ "column_def_list ::= column_def", - /* 158 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 159 */ "column_def ::= column_name type_name", - /* 160 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 161 */ "type_name ::= BOOL", - /* 162 */ "type_name ::= TINYINT", - /* 163 */ "type_name ::= SMALLINT", - /* 164 */ "type_name ::= INT", - /* 165 */ "type_name ::= INTEGER", - /* 166 */ "type_name ::= BIGINT", - /* 167 */ "type_name ::= FLOAT", - /* 168 */ "type_name ::= DOUBLE", - /* 169 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 170 */ "type_name ::= TIMESTAMP", - /* 171 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 172 */ "type_name ::= TINYINT UNSIGNED", - /* 173 */ "type_name ::= SMALLINT UNSIGNED", - /* 174 */ "type_name ::= INT UNSIGNED", - /* 175 */ "type_name ::= BIGINT UNSIGNED", - /* 176 */ "type_name ::= JSON", - /* 177 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 178 */ "type_name ::= MEDIUMBLOB", - /* 179 */ "type_name ::= BLOB", - /* 180 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 181 */ "type_name ::= DECIMAL", - /* 182 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 183 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 184 */ "tags_def_opt ::=", - /* 185 */ "tags_def_opt ::= tags_def", - /* 186 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 187 */ "table_options ::=", - /* 188 */ "table_options ::= table_options COMMENT NK_STRING", - /* 189 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 190 */ "table_options ::= table_options WATERMARK duration_list", - /* 191 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 192 */ "table_options ::= table_options TTL NK_INTEGER", - /* 193 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 194 */ "alter_table_options ::= alter_table_option", - /* 195 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 196 */ "alter_table_option ::= COMMENT NK_STRING", - /* 197 */ "alter_table_option ::= TTL NK_INTEGER", - /* 198 */ "duration_list ::= duration_literal", - /* 199 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 200 */ "rollup_func_list ::= rollup_func_name", - /* 201 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 202 */ "rollup_func_name ::= function_name", - /* 203 */ "rollup_func_name ::= FIRST", - /* 204 */ "rollup_func_name ::= LAST", - /* 205 */ "col_name_list ::= col_name", - /* 206 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 207 */ "col_name ::= column_name", - /* 208 */ "cmd ::= SHOW DNODES", - /* 209 */ "cmd ::= SHOW USERS", - /* 210 */ "cmd ::= SHOW USER PRIVILEGES", - /* 211 */ "cmd ::= SHOW DATABASES", - /* 212 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 213 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 214 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 215 */ "cmd ::= SHOW MNODES", - /* 216 */ "cmd ::= SHOW QNODES", - /* 217 */ "cmd ::= SHOW FUNCTIONS", - /* 218 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 219 */ "cmd ::= SHOW STREAMS", - /* 220 */ "cmd ::= SHOW ACCOUNTS", - /* 221 */ "cmd ::= SHOW APPS", - /* 222 */ "cmd ::= SHOW CONNECTIONS", - /* 223 */ "cmd ::= SHOW LICENCES", - /* 224 */ "cmd ::= SHOW GRANTS", - /* 225 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 226 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 227 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 228 */ "cmd ::= SHOW QUERIES", - /* 229 */ "cmd ::= SHOW SCORES", - /* 230 */ "cmd ::= SHOW TOPICS", - /* 231 */ "cmd ::= SHOW VARIABLES", - /* 232 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 233 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 234 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 235 */ "cmd ::= SHOW BNODES", - /* 236 */ "cmd ::= SHOW SNODES", - /* 237 */ "cmd ::= SHOW CLUSTER", - /* 238 */ "cmd ::= SHOW TRANSACTIONS", - /* 239 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 240 */ "cmd ::= SHOW CONSUMERS", - /* 241 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 242 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 243 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 244 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 245 */ "cmd ::= SHOW VNODES NK_STRING", - /* 246 */ "db_name_cond_opt ::=", - /* 247 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 248 */ "like_pattern_opt ::=", - /* 249 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 250 */ "table_name_cond ::= table_name", - /* 251 */ "from_db_opt ::=", - /* 252 */ "from_db_opt ::= FROM db_name", - /* 253 */ "tag_list_opt ::=", - /* 254 */ "tag_list_opt ::= tag_item", - /* 255 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 256 */ "tag_item ::= TBNAME", - /* 257 */ "tag_item ::= QTAGS", - /* 258 */ "tag_item ::= column_name", - /* 259 */ "tag_item ::= column_name column_alias", - /* 260 */ "tag_item ::= column_name AS column_alias", - /* 261 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", - /* 262 */ "cmd ::= DROP INDEX exists_opt full_table_name", - /* 263 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 264 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 265 */ "func_list ::= func", - /* 266 */ "func_list ::= func_list NK_COMMA func", - /* 267 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 268 */ "sma_stream_opt ::=", - /* 269 */ "sma_stream_opt ::= stream_options WATERMARK duration_literal", - /* 270 */ "sma_stream_opt ::= stream_options MAX_DELAY duration_literal", + /* 92 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 93 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 94 */ "db_options ::= db_options RETENTIONS retention_list", + /* 95 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 96 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 97 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 98 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 100 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 102 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 103 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 104 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 105 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", + /* 106 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 107 */ "alter_db_options ::= alter_db_option", + /* 108 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 109 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 110 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 111 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 112 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 113 */ "alter_db_option ::= KEEP integer_list", + /* 114 */ "alter_db_option ::= KEEP variable_list", + /* 115 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 116 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 117 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 118 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 119 */ "integer_list ::= NK_INTEGER", + /* 120 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 121 */ "variable_list ::= NK_VARIABLE", + /* 122 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 123 */ "retention_list ::= retention", + /* 124 */ "retention_list ::= retention_list NK_COMMA retention", + /* 125 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 126 */ "speed_opt ::=", + /* 127 */ "speed_opt ::= MAX_SPEED NK_INTEGER", + /* 128 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 129 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 130 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 131 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 132 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 133 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 134 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 135 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 136 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 137 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 138 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 139 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 140 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 141 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 142 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 143 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 144 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 145 */ "multi_create_clause ::= create_subtable_clause", + /* 146 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 147 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 148 */ "multi_drop_clause ::= drop_table_clause", + /* 149 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 150 */ "drop_table_clause ::= exists_opt full_table_name", + /* 151 */ "specific_cols_opt ::=", + /* 152 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 153 */ "full_table_name ::= table_name", + /* 154 */ "full_table_name ::= db_name NK_DOT table_name", + /* 155 */ "column_def_list ::= column_def", + /* 156 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 157 */ "column_def ::= column_name type_name", + /* 158 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 159 */ "type_name ::= BOOL", + /* 160 */ "type_name ::= TINYINT", + /* 161 */ "type_name ::= SMALLINT", + /* 162 */ "type_name ::= INT", + /* 163 */ "type_name ::= INTEGER", + /* 164 */ "type_name ::= BIGINT", + /* 165 */ "type_name ::= FLOAT", + /* 166 */ "type_name ::= DOUBLE", + /* 167 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 168 */ "type_name ::= TIMESTAMP", + /* 169 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 170 */ "type_name ::= TINYINT UNSIGNED", + /* 171 */ "type_name ::= SMALLINT UNSIGNED", + /* 172 */ "type_name ::= INT UNSIGNED", + /* 173 */ "type_name ::= BIGINT UNSIGNED", + /* 174 */ "type_name ::= JSON", + /* 175 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 176 */ "type_name ::= MEDIUMBLOB", + /* 177 */ "type_name ::= BLOB", + /* 178 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 179 */ "type_name ::= DECIMAL", + /* 180 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 181 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 182 */ "tags_def_opt ::=", + /* 183 */ "tags_def_opt ::= tags_def", + /* 184 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 185 */ "table_options ::=", + /* 186 */ "table_options ::= table_options COMMENT NK_STRING", + /* 187 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 188 */ "table_options ::= table_options WATERMARK duration_list", + /* 189 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 190 */ "table_options ::= table_options TTL NK_INTEGER", + /* 191 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 192 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 193 */ "alter_table_options ::= alter_table_option", + /* 194 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 195 */ "alter_table_option ::= COMMENT NK_STRING", + /* 196 */ "alter_table_option ::= TTL NK_INTEGER", + /* 197 */ "duration_list ::= duration_literal", + /* 198 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 199 */ "rollup_func_list ::= rollup_func_name", + /* 200 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 201 */ "rollup_func_name ::= function_name", + /* 202 */ "rollup_func_name ::= FIRST", + /* 203 */ "rollup_func_name ::= LAST", + /* 204 */ "col_name_list ::= col_name", + /* 205 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 206 */ "col_name ::= column_name", + /* 207 */ "cmd ::= SHOW DNODES", + /* 208 */ "cmd ::= SHOW USERS", + /* 209 */ "cmd ::= SHOW USER PRIVILEGES", + /* 210 */ "cmd ::= SHOW DATABASES", + /* 211 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 212 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 213 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 214 */ "cmd ::= SHOW MNODES", + /* 215 */ "cmd ::= SHOW QNODES", + /* 216 */ "cmd ::= SHOW FUNCTIONS", + /* 217 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 218 */ "cmd ::= SHOW STREAMS", + /* 219 */ "cmd ::= SHOW ACCOUNTS", + /* 220 */ "cmd ::= SHOW APPS", + /* 221 */ "cmd ::= SHOW CONNECTIONS", + /* 222 */ "cmd ::= SHOW LICENCES", + /* 223 */ "cmd ::= SHOW GRANTS", + /* 224 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 225 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 226 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 227 */ "cmd ::= SHOW QUERIES", + /* 228 */ "cmd ::= SHOW SCORES", + /* 229 */ "cmd ::= SHOW TOPICS", + /* 230 */ "cmd ::= SHOW VARIABLES", + /* 231 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 232 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 233 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 234 */ "cmd ::= SHOW BNODES", + /* 235 */ "cmd ::= SHOW SNODES", + /* 236 */ "cmd ::= SHOW CLUSTER", + /* 237 */ "cmd ::= SHOW TRANSACTIONS", + /* 238 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 239 */ "cmd ::= SHOW CONSUMERS", + /* 240 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 241 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 242 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 243 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 244 */ "cmd ::= SHOW VNODES NK_STRING", + /* 245 */ "db_name_cond_opt ::=", + /* 246 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 247 */ "like_pattern_opt ::=", + /* 248 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 249 */ "table_name_cond ::= table_name", + /* 250 */ "from_db_opt ::=", + /* 251 */ "from_db_opt ::= FROM db_name", + /* 252 */ "tag_list_opt ::=", + /* 253 */ "tag_list_opt ::= tag_item", + /* 254 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 255 */ "tag_item ::= TBNAME", + /* 256 */ "tag_item ::= QTAGS", + /* 257 */ "tag_item ::= column_name", + /* 258 */ "tag_item ::= column_name column_alias", + /* 259 */ "tag_item ::= column_name AS column_alias", + /* 260 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", + /* 261 */ "cmd ::= DROP INDEX exists_opt full_table_name", + /* 262 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 263 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 264 */ "func_list ::= func", + /* 265 */ "func_list ::= func_list NK_COMMA func", + /* 266 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 267 */ "sma_stream_opt ::=", + /* 268 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 269 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 270 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", /* 271 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", /* 272 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", /* 273 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", @@ -2290,57 +2383,58 @@ static const char *const yyRuleName[] = { /* 484 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", /* 485 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", /* 486 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 487 */ "sliding_opt ::=", - /* 488 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 489 */ "fill_opt ::=", - /* 490 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 491 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 492 */ "fill_mode ::= NONE", - /* 493 */ "fill_mode ::= PREV", - /* 494 */ "fill_mode ::= NULL", - /* 495 */ "fill_mode ::= LINEAR", - /* 496 */ "fill_mode ::= NEXT", - /* 497 */ "group_by_clause_opt ::=", - /* 498 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 499 */ "group_by_list ::= expr_or_subquery", - /* 500 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 501 */ "having_clause_opt ::=", - /* 502 */ "having_clause_opt ::= HAVING search_condition", - /* 503 */ "range_opt ::=", - /* 504 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 505 */ "every_opt ::=", - /* 506 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 507 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 508 */ "query_simple ::= query_specification", - /* 509 */ "query_simple ::= union_query_expression", - /* 510 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 511 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 512 */ "query_simple_or_subquery ::= query_simple", - /* 513 */ "query_simple_or_subquery ::= subquery", - /* 514 */ "query_or_subquery ::= query_expression", - /* 515 */ "query_or_subquery ::= subquery", - /* 516 */ "order_by_clause_opt ::=", - /* 517 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 518 */ "slimit_clause_opt ::=", - /* 519 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 520 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 521 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 522 */ "limit_clause_opt ::=", - /* 523 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 524 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 525 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 526 */ "subquery ::= NK_LP query_expression NK_RP", - /* 527 */ "subquery ::= NK_LP subquery NK_RP", - /* 528 */ "search_condition ::= common_expression", - /* 529 */ "sort_specification_list ::= sort_specification", - /* 530 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 531 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 532 */ "ordering_specification_opt ::=", - /* 533 */ "ordering_specification_opt ::= ASC", - /* 534 */ "ordering_specification_opt ::= DESC", - /* 535 */ "null_ordering_opt ::=", - /* 536 */ "null_ordering_opt ::= NULLS FIRST", - /* 537 */ "null_ordering_opt ::= NULLS LAST", + /* 487 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 488 */ "sliding_opt ::=", + /* 489 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 490 */ "fill_opt ::=", + /* 491 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 492 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 493 */ "fill_mode ::= NONE", + /* 494 */ "fill_mode ::= PREV", + /* 495 */ "fill_mode ::= NULL", + /* 496 */ "fill_mode ::= LINEAR", + /* 497 */ "fill_mode ::= NEXT", + /* 498 */ "group_by_clause_opt ::=", + /* 499 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 500 */ "group_by_list ::= expr_or_subquery", + /* 501 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 502 */ "having_clause_opt ::=", + /* 503 */ "having_clause_opt ::= HAVING search_condition", + /* 504 */ "range_opt ::=", + /* 505 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 506 */ "every_opt ::=", + /* 507 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 508 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 509 */ "query_simple ::= query_specification", + /* 510 */ "query_simple ::= union_query_expression", + /* 511 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 512 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 513 */ "query_simple_or_subquery ::= query_simple", + /* 514 */ "query_simple_or_subquery ::= subquery", + /* 515 */ "query_or_subquery ::= query_expression", + /* 516 */ "query_or_subquery ::= subquery", + /* 517 */ "order_by_clause_opt ::=", + /* 518 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 519 */ "slimit_clause_opt ::=", + /* 520 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 521 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 522 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 523 */ "limit_clause_opt ::=", + /* 524 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 525 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 526 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 527 */ "subquery ::= NK_LP query_expression NK_RP", + /* 528 */ "subquery ::= NK_LP subquery NK_RP", + /* 529 */ "search_condition ::= common_expression", + /* 530 */ "sort_specification_list ::= sort_specification", + /* 531 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 532 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 533 */ "ordering_specification_opt ::=", + /* 534 */ "ordering_specification_opt ::= ASC", + /* 535 */ "ordering_specification_opt ::= DESC", + /* 536 */ "null_ordering_opt ::=", + /* 537 */ "null_ordering_opt ::= NULLS FIRST", + /* 538 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2467,193 +2561,193 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 321: /* cmd */ - case 324: /* literal */ - case 337: /* db_options */ - case 339: /* alter_db_options */ - case 345: /* retention */ - case 346: /* full_table_name */ - case 349: /* table_options */ - case 353: /* alter_table_clause */ - case 354: /* alter_table_options */ - case 357: /* signed_literal */ - case 358: /* create_subtable_clause */ - case 361: /* drop_table_clause */ - case 364: /* column_def */ - case 368: /* duration_literal */ - case 369: /* rollup_func_name */ - case 371: /* col_name */ - case 372: /* db_name_cond_opt */ - case 373: /* like_pattern_opt */ - case 374: /* table_name_cond */ - case 375: /* from_db_opt */ - case 377: /* tag_item */ - case 379: /* index_options */ - case 381: /* sliding_opt */ - case 382: /* sma_stream_opt */ - case 383: /* func */ - case 384: /* stream_options */ - case 385: /* query_or_subquery */ - case 388: /* explain_options */ - case 392: /* subtable_opt */ - case 393: /* expression */ - case 395: /* where_clause_opt */ - case 396: /* signed */ - case 397: /* literal_func */ - case 400: /* expr_or_subquery */ - case 401: /* pseudo_column */ - case 402: /* column_reference */ - case 403: /* function_expression */ - case 404: /* case_when_expression */ - case 409: /* star_func_para */ - case 411: /* case_when_else_opt */ - case 412: /* common_expression */ - case 413: /* when_then_expr */ - case 414: /* predicate */ - case 417: /* in_predicate_value */ - case 418: /* boolean_value_expression */ - case 419: /* boolean_primary */ - case 420: /* from_clause_opt */ - case 421: /* table_reference_list */ - case 422: /* table_reference */ - case 423: /* table_primary */ - case 424: /* joined_table */ - case 426: /* subquery */ - case 427: /* parenthesized_joined_table */ - case 429: /* search_condition */ - case 430: /* query_specification */ - case 434: /* range_opt */ - case 435: /* every_opt */ - case 436: /* fill_opt */ - case 437: /* twindow_clause_opt */ - case 439: /* having_clause_opt */ - case 440: /* select_item */ - case 442: /* partition_item */ - case 445: /* query_expression */ - case 446: /* query_simple */ - case 448: /* slimit_clause_opt */ - case 449: /* limit_clause_opt */ - case 450: /* union_query_expression */ - case 451: /* query_simple_or_subquery */ - case 453: /* sort_specification */ + case 324: /* cmd */ + case 327: /* literal */ + case 340: /* db_options */ + case 342: /* alter_db_options */ + case 348: /* retention */ + case 349: /* full_table_name */ + case 352: /* table_options */ + case 356: /* alter_table_clause */ + case 357: /* alter_table_options */ + case 360: /* signed_literal */ + case 361: /* create_subtable_clause */ + case 364: /* drop_table_clause */ + case 367: /* column_def */ + case 371: /* duration_literal */ + case 372: /* rollup_func_name */ + case 374: /* col_name */ + case 375: /* db_name_cond_opt */ + case 376: /* like_pattern_opt */ + case 377: /* table_name_cond */ + case 378: /* from_db_opt */ + case 380: /* tag_item */ + case 382: /* index_options */ + case 384: /* sliding_opt */ + case 385: /* sma_stream_opt */ + case 386: /* func */ + case 387: /* query_or_subquery */ + case 390: /* explain_options */ + case 394: /* stream_options */ + case 395: /* subtable_opt */ + case 396: /* expression */ + case 398: /* where_clause_opt */ + case 399: /* signed */ + case 400: /* literal_func */ + case 403: /* expr_or_subquery */ + case 404: /* pseudo_column */ + case 405: /* column_reference */ + case 406: /* function_expression */ + case 407: /* case_when_expression */ + case 412: /* star_func_para */ + case 414: /* case_when_else_opt */ + case 415: /* common_expression */ + case 416: /* when_then_expr */ + case 417: /* predicate */ + case 420: /* in_predicate_value */ + case 421: /* boolean_value_expression */ + case 422: /* boolean_primary */ + case 423: /* from_clause_opt */ + case 424: /* table_reference_list */ + case 425: /* table_reference */ + case 426: /* table_primary */ + case 427: /* joined_table */ + case 429: /* subquery */ + case 430: /* parenthesized_joined_table */ + case 432: /* search_condition */ + case 433: /* query_specification */ + case 437: /* range_opt */ + case 438: /* every_opt */ + case 439: /* fill_opt */ + case 440: /* twindow_clause_opt */ + case 442: /* having_clause_opt */ + case 443: /* select_item */ + case 445: /* partition_item */ + case 448: /* query_expression */ + case 449: /* query_simple */ + case 451: /* slimit_clause_opt */ + case 452: /* limit_clause_opt */ + case 453: /* union_query_expression */ + case 454: /* query_simple_or_subquery */ + case 456: /* sort_specification */ { - nodesDestroyNode((yypminor->yy104)); + nodesDestroyNode((yypminor->yy74)); } break; - case 322: /* account_options */ - case 323: /* alter_account_options */ - case 325: /* alter_account_option */ - case 340: /* speed_opt */ - case 390: /* bufsize_opt */ + case 325: /* account_options */ + case 326: /* alter_account_options */ + case 328: /* alter_account_option */ + case 343: /* speed_opt */ + case 392: /* bufsize_opt */ { } break; - case 326: /* user_name */ - case 329: /* priv_level */ - case 332: /* db_name */ - case 333: /* topic_name */ - case 334: /* dnode_endpoint */ - case 355: /* column_name */ - case 363: /* table_name */ - case 370: /* function_name */ - case 378: /* column_alias */ - case 386: /* cgroup_name */ - case 391: /* stream_name */ - case 399: /* table_alias */ - case 405: /* star_func */ - case 407: /* noarg_func */ - case 425: /* alias_opt */ + case 329: /* user_name */ + case 332: /* priv_level */ + case 335: /* db_name */ + case 336: /* topic_name */ + case 337: /* dnode_endpoint */ + case 358: /* column_name */ + case 366: /* table_name */ + case 373: /* function_name */ + case 381: /* column_alias */ + case 388: /* cgroup_name */ + case 393: /* stream_name */ + case 402: /* table_alias */ + case 408: /* star_func */ + case 410: /* noarg_func */ + case 428: /* alias_opt */ { } break; - case 327: /* sysinfo_opt */ + case 330: /* sysinfo_opt */ { } break; - case 328: /* privileges */ - case 330: /* priv_type_list */ - case 331: /* priv_type */ + case 331: /* privileges */ + case 333: /* priv_type_list */ + case 334: /* priv_type */ { } break; - case 335: /* force_opt */ - case 336: /* not_exists_opt */ - case 338: /* exists_opt */ - case 387: /* analyze_opt */ - case 389: /* agg_func_opt */ - case 431: /* set_quantifier_opt */ + case 338: /* force_opt */ + case 339: /* not_exists_opt */ + case 341: /* exists_opt */ + case 389: /* analyze_opt */ + case 391: /* agg_func_opt */ + case 434: /* set_quantifier_opt */ { } break; - case 341: /* integer_list */ - case 342: /* variable_list */ - case 343: /* retention_list */ - case 347: /* column_def_list */ - case 348: /* tags_def_opt */ - case 350: /* multi_create_clause */ - case 351: /* tags_def */ - case 352: /* multi_drop_clause */ - case 359: /* specific_cols_opt */ - case 360: /* expression_list */ - case 362: /* col_name_list */ - case 365: /* duration_list */ - case 366: /* rollup_func_list */ - case 376: /* tag_list_opt */ - case 380: /* func_list */ - case 394: /* dnode_list */ - case 398: /* literal_list */ - case 406: /* star_func_para_list */ - case 408: /* other_para_list */ - case 410: /* when_then_list */ - case 432: /* select_list */ - case 433: /* partition_by_clause_opt */ - case 438: /* group_by_clause_opt */ - case 441: /* partition_list */ - case 444: /* group_by_list */ - case 447: /* order_by_clause_opt */ - case 452: /* sort_specification_list */ + case 344: /* integer_list */ + case 345: /* variable_list */ + case 346: /* retention_list */ + case 350: /* column_def_list */ + case 351: /* tags_def_opt */ + case 353: /* multi_create_clause */ + case 354: /* tags_def */ + case 355: /* multi_drop_clause */ + case 362: /* specific_cols_opt */ + case 363: /* expression_list */ + case 365: /* col_name_list */ + case 368: /* duration_list */ + case 369: /* rollup_func_list */ + case 379: /* tag_list_opt */ + case 383: /* func_list */ + case 397: /* dnode_list */ + case 401: /* literal_list */ + case 409: /* star_func_para_list */ + case 411: /* other_para_list */ + case 413: /* when_then_list */ + case 435: /* select_list */ + case 436: /* partition_by_clause_opt */ + case 441: /* group_by_clause_opt */ + case 444: /* partition_list */ + case 447: /* group_by_list */ + case 450: /* order_by_clause_opt */ + case 455: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy616)); + nodesDestroyList((yypminor->yy874)); } break; - case 344: /* alter_db_option */ - case 367: /* alter_table_option */ + case 347: /* alter_db_option */ + case 370: /* alter_table_option */ { } break; - case 356: /* type_name */ + case 359: /* type_name */ { } break; - case 415: /* compare_op */ - case 416: /* in_op */ + case 418: /* compare_op */ + case 419: /* in_op */ { } break; - case 428: /* join_type */ + case 431: /* join_type */ { } break; - case 443: /* fill_mode */ + case 446: /* fill_mode */ { } break; - case 454: /* ordering_specification_opt */ + case 457: /* ordering_specification_opt */ { } break; - case 455: /* null_ordering_opt */ + case 458: /* null_ordering_opt */ { } @@ -2952,544 +3046,545 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 321, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 321, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 322, 0 }, /* (2) account_options ::= */ - { 322, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 322, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 322, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 322, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 322, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 322, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 322, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 322, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 322, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 323, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 323, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 325, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 325, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 325, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 325, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 325, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 325, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 325, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 325, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 325, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 325, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 321, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - { 321, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 321, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - { 321, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - { 321, -3 }, /* (28) cmd ::= DROP USER user_name */ - { 327, 0 }, /* (29) sysinfo_opt ::= */ - { 327, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 321, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 321, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 328, -1 }, /* (33) privileges ::= ALL */ - { 328, -1 }, /* (34) privileges ::= priv_type_list */ - { 328, -1 }, /* (35) privileges ::= SUBSCRIBE */ - { 330, -1 }, /* (36) priv_type_list ::= priv_type */ - { 330, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 331, -1 }, /* (38) priv_type ::= READ */ - { 331, -1 }, /* (39) priv_type ::= WRITE */ - { 329, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 329, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - { 329, -1 }, /* (42) priv_level ::= topic_name */ - { 321, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ - { 321, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 321, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ - { 321, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ - { 321, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 321, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 321, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ - { 321, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 334, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ - { 334, -1 }, /* (52) dnode_endpoint ::= NK_ID */ - { 334, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ - { 335, 0 }, /* (54) force_opt ::= */ - { 335, -1 }, /* (55) force_opt ::= FORCE */ - { 321, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ - { 321, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 321, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 321, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 321, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ - { 321, -2 }, /* (68) cmd ::= USE db_name */ - { 321, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 321, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ - { 321, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ - { 336, -3 }, /* (72) not_exists_opt ::= IF NOT EXISTS */ - { 336, 0 }, /* (73) not_exists_opt ::= */ - { 338, -2 }, /* (74) exists_opt ::= IF EXISTS */ - { 338, 0 }, /* (75) exists_opt ::= */ - { 337, 0 }, /* (76) db_options ::= */ - { 337, -3 }, /* (77) db_options ::= db_options BUFFER NK_INTEGER */ - { 337, -3 }, /* (78) db_options ::= db_options CACHEMODEL NK_STRING */ - { 337, -3 }, /* (79) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 337, -3 }, /* (80) db_options ::= db_options COMP NK_INTEGER */ - { 337, -3 }, /* (81) db_options ::= db_options DURATION NK_INTEGER */ - { 337, -3 }, /* (82) db_options ::= db_options DURATION NK_VARIABLE */ - { 337, -3 }, /* (83) db_options ::= db_options MAXROWS NK_INTEGER */ - { 337, -3 }, /* (84) db_options ::= db_options MINROWS NK_INTEGER */ - { 337, -3 }, /* (85) db_options ::= db_options KEEP integer_list */ - { 337, -3 }, /* (86) db_options ::= db_options KEEP variable_list */ - { 337, -3 }, /* (87) db_options ::= db_options PAGES NK_INTEGER */ - { 337, -3 }, /* (88) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 337, -3 }, /* (89) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - { 337, -3 }, /* (90) db_options ::= db_options PRECISION NK_STRING */ - { 337, -3 }, /* (91) db_options ::= db_options REPLICA NK_INTEGER */ - { 337, -3 }, /* (92) db_options ::= db_options STRICT NK_STRING */ - { 337, -3 }, /* (93) db_options ::= db_options VGROUPS NK_INTEGER */ - { 337, -3 }, /* (94) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 337, -3 }, /* (95) db_options ::= db_options RETENTIONS retention_list */ - { 337, -3 }, /* (96) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 337, -3 }, /* (97) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 337, -3 }, /* (98) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 337, -3 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 337, -4 }, /* (100) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 337, -3 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 337, -4 }, /* (102) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 337, -3 }, /* (103) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 337, -3 }, /* (104) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 337, -3 }, /* (105) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - { 337, -3 }, /* (106) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ - { 337, -3 }, /* (107) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ - { 339, -1 }, /* (108) alter_db_options ::= alter_db_option */ - { 339, -2 }, /* (109) alter_db_options ::= alter_db_options alter_db_option */ - { 344, -2 }, /* (110) alter_db_option ::= BUFFER NK_INTEGER */ - { 344, -2 }, /* (111) alter_db_option ::= CACHEMODEL NK_STRING */ - { 344, -2 }, /* (112) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 344, -2 }, /* (113) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 344, -2 }, /* (114) alter_db_option ::= KEEP integer_list */ - { 344, -2 }, /* (115) alter_db_option ::= KEEP variable_list */ - { 344, -2 }, /* (116) alter_db_option ::= PAGES NK_INTEGER */ - { 344, -2 }, /* (117) alter_db_option ::= REPLICA NK_INTEGER */ - { 344, -2 }, /* (118) alter_db_option ::= STRICT NK_STRING */ - { 344, -2 }, /* (119) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 344, -2 }, /* (120) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - { 341, -1 }, /* (121) integer_list ::= NK_INTEGER */ - { 341, -3 }, /* (122) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 342, -1 }, /* (123) variable_list ::= NK_VARIABLE */ - { 342, -3 }, /* (124) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 343, -1 }, /* (125) retention_list ::= retention */ - { 343, -3 }, /* (126) retention_list ::= retention_list NK_COMMA retention */ - { 345, -3 }, /* (127) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 340, 0 }, /* (128) speed_opt ::= */ - { 340, -2 }, /* (129) speed_opt ::= MAX_SPEED NK_INTEGER */ - { 321, -9 }, /* (130) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 321, -3 }, /* (131) cmd ::= CREATE TABLE multi_create_clause */ - { 321, -9 }, /* (132) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 321, -3 }, /* (133) cmd ::= DROP TABLE multi_drop_clause */ - { 321, -4 }, /* (134) cmd ::= DROP STABLE exists_opt full_table_name */ - { 321, -3 }, /* (135) cmd ::= ALTER TABLE alter_table_clause */ - { 321, -3 }, /* (136) cmd ::= ALTER STABLE alter_table_clause */ - { 353, -2 }, /* (137) alter_table_clause ::= full_table_name alter_table_options */ - { 353, -5 }, /* (138) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 353, -4 }, /* (139) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 353, -5 }, /* (140) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 353, -5 }, /* (141) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 353, -5 }, /* (142) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 353, -4 }, /* (143) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 353, -5 }, /* (144) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 353, -5 }, /* (145) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 353, -6 }, /* (146) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 350, -1 }, /* (147) multi_create_clause ::= create_subtable_clause */ - { 350, -2 }, /* (148) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 358, -10 }, /* (149) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 352, -1 }, /* (150) multi_drop_clause ::= drop_table_clause */ - { 352, -2 }, /* (151) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 361, -2 }, /* (152) drop_table_clause ::= exists_opt full_table_name */ - { 359, 0 }, /* (153) specific_cols_opt ::= */ - { 359, -3 }, /* (154) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 346, -1 }, /* (155) full_table_name ::= table_name */ - { 346, -3 }, /* (156) full_table_name ::= db_name NK_DOT table_name */ - { 347, -1 }, /* (157) column_def_list ::= column_def */ - { 347, -3 }, /* (158) column_def_list ::= column_def_list NK_COMMA column_def */ - { 364, -2 }, /* (159) column_def ::= column_name type_name */ - { 364, -4 }, /* (160) column_def ::= column_name type_name COMMENT NK_STRING */ - { 356, -1 }, /* (161) type_name ::= BOOL */ - { 356, -1 }, /* (162) type_name ::= TINYINT */ - { 356, -1 }, /* (163) type_name ::= SMALLINT */ - { 356, -1 }, /* (164) type_name ::= INT */ - { 356, -1 }, /* (165) type_name ::= INTEGER */ - { 356, -1 }, /* (166) type_name ::= BIGINT */ - { 356, -1 }, /* (167) type_name ::= FLOAT */ - { 356, -1 }, /* (168) type_name ::= DOUBLE */ - { 356, -4 }, /* (169) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 356, -1 }, /* (170) type_name ::= TIMESTAMP */ - { 356, -4 }, /* (171) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 356, -2 }, /* (172) type_name ::= TINYINT UNSIGNED */ - { 356, -2 }, /* (173) type_name ::= SMALLINT UNSIGNED */ - { 356, -2 }, /* (174) type_name ::= INT UNSIGNED */ - { 356, -2 }, /* (175) type_name ::= BIGINT UNSIGNED */ - { 356, -1 }, /* (176) type_name ::= JSON */ - { 356, -4 }, /* (177) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 356, -1 }, /* (178) type_name ::= MEDIUMBLOB */ - { 356, -1 }, /* (179) type_name ::= BLOB */ - { 356, -4 }, /* (180) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 356, -1 }, /* (181) type_name ::= DECIMAL */ - { 356, -4 }, /* (182) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 356, -6 }, /* (183) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 348, 0 }, /* (184) tags_def_opt ::= */ - { 348, -1 }, /* (185) tags_def_opt ::= tags_def */ - { 351, -4 }, /* (186) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 349, 0 }, /* (187) table_options ::= */ - { 349, -3 }, /* (188) table_options ::= table_options COMMENT NK_STRING */ - { 349, -3 }, /* (189) table_options ::= table_options MAX_DELAY duration_list */ - { 349, -3 }, /* (190) table_options ::= table_options WATERMARK duration_list */ - { 349, -5 }, /* (191) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 349, -3 }, /* (192) table_options ::= table_options TTL NK_INTEGER */ - { 349, -5 }, /* (193) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 354, -1 }, /* (194) alter_table_options ::= alter_table_option */ - { 354, -2 }, /* (195) alter_table_options ::= alter_table_options alter_table_option */ - { 367, -2 }, /* (196) alter_table_option ::= COMMENT NK_STRING */ - { 367, -2 }, /* (197) alter_table_option ::= TTL NK_INTEGER */ - { 365, -1 }, /* (198) duration_list ::= duration_literal */ - { 365, -3 }, /* (199) duration_list ::= duration_list NK_COMMA duration_literal */ - { 366, -1 }, /* (200) rollup_func_list ::= rollup_func_name */ - { 366, -3 }, /* (201) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 369, -1 }, /* (202) rollup_func_name ::= function_name */ - { 369, -1 }, /* (203) rollup_func_name ::= FIRST */ - { 369, -1 }, /* (204) rollup_func_name ::= LAST */ - { 362, -1 }, /* (205) col_name_list ::= col_name */ - { 362, -3 }, /* (206) col_name_list ::= col_name_list NK_COMMA col_name */ - { 371, -1 }, /* (207) col_name ::= column_name */ - { 321, -2 }, /* (208) cmd ::= SHOW DNODES */ - { 321, -2 }, /* (209) cmd ::= SHOW USERS */ - { 321, -3 }, /* (210) cmd ::= SHOW USER PRIVILEGES */ - { 321, -2 }, /* (211) cmd ::= SHOW DATABASES */ - { 321, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 321, -4 }, /* (213) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 321, -3 }, /* (214) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 321, -2 }, /* (215) cmd ::= SHOW MNODES */ - { 321, -2 }, /* (216) cmd ::= SHOW QNODES */ - { 321, -2 }, /* (217) cmd ::= SHOW FUNCTIONS */ - { 321, -5 }, /* (218) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 321, -2 }, /* (219) cmd ::= SHOW STREAMS */ - { 321, -2 }, /* (220) cmd ::= SHOW ACCOUNTS */ - { 321, -2 }, /* (221) cmd ::= SHOW APPS */ - { 321, -2 }, /* (222) cmd ::= SHOW CONNECTIONS */ - { 321, -2 }, /* (223) cmd ::= SHOW LICENCES */ - { 321, -2 }, /* (224) cmd ::= SHOW GRANTS */ - { 321, -4 }, /* (225) cmd ::= SHOW CREATE DATABASE db_name */ - { 321, -4 }, /* (226) cmd ::= SHOW CREATE TABLE full_table_name */ - { 321, -4 }, /* (227) cmd ::= SHOW CREATE STABLE full_table_name */ - { 321, -2 }, /* (228) cmd ::= SHOW QUERIES */ - { 321, -2 }, /* (229) cmd ::= SHOW SCORES */ - { 321, -2 }, /* (230) cmd ::= SHOW TOPICS */ - { 321, -2 }, /* (231) cmd ::= SHOW VARIABLES */ - { 321, -3 }, /* (232) cmd ::= SHOW CLUSTER VARIABLES */ - { 321, -3 }, /* (233) cmd ::= SHOW LOCAL VARIABLES */ - { 321, -5 }, /* (234) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 321, -2 }, /* (235) cmd ::= SHOW BNODES */ - { 321, -2 }, /* (236) cmd ::= SHOW SNODES */ - { 321, -2 }, /* (237) cmd ::= SHOW CLUSTER */ - { 321, -2 }, /* (238) cmd ::= SHOW TRANSACTIONS */ - { 321, -4 }, /* (239) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 321, -2 }, /* (240) cmd ::= SHOW CONSUMERS */ - { 321, -2 }, /* (241) cmd ::= SHOW SUBSCRIPTIONS */ - { 321, -5 }, /* (242) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 321, -7 }, /* (243) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 321, -3 }, /* (244) cmd ::= SHOW VNODES NK_INTEGER */ - { 321, -3 }, /* (245) cmd ::= SHOW VNODES NK_STRING */ - { 372, 0 }, /* (246) db_name_cond_opt ::= */ - { 372, -2 }, /* (247) db_name_cond_opt ::= db_name NK_DOT */ - { 373, 0 }, /* (248) like_pattern_opt ::= */ - { 373, -2 }, /* (249) like_pattern_opt ::= LIKE NK_STRING */ - { 374, -1 }, /* (250) table_name_cond ::= table_name */ - { 375, 0 }, /* (251) from_db_opt ::= */ - { 375, -2 }, /* (252) from_db_opt ::= FROM db_name */ - { 376, 0 }, /* (253) tag_list_opt ::= */ - { 376, -1 }, /* (254) tag_list_opt ::= tag_item */ - { 376, -3 }, /* (255) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 377, -1 }, /* (256) tag_item ::= TBNAME */ - { 377, -1 }, /* (257) tag_item ::= QTAGS */ - { 377, -1 }, /* (258) tag_item ::= column_name */ - { 377, -2 }, /* (259) tag_item ::= column_name column_alias */ - { 377, -3 }, /* (260) tag_item ::= column_name AS column_alias */ - { 321, -8 }, /* (261) cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ - { 321, -4 }, /* (262) cmd ::= DROP INDEX exists_opt full_table_name */ - { 379, -10 }, /* (263) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 379, -12 }, /* (264) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 380, -1 }, /* (265) func_list ::= func */ - { 380, -3 }, /* (266) func_list ::= func_list NK_COMMA func */ - { 383, -4 }, /* (267) func ::= function_name NK_LP expression_list NK_RP */ - { 382, 0 }, /* (268) sma_stream_opt ::= */ - { 382, -3 }, /* (269) sma_stream_opt ::= stream_options WATERMARK duration_literal */ - { 382, -3 }, /* (270) sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ - { 321, -6 }, /* (271) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 321, -7 }, /* (272) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 321, -9 }, /* (273) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 321, -7 }, /* (274) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 321, -9 }, /* (275) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 321, -4 }, /* (276) cmd ::= DROP TOPIC exists_opt topic_name */ - { 321, -7 }, /* (277) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 321, -2 }, /* (278) cmd ::= DESC full_table_name */ - { 321, -2 }, /* (279) cmd ::= DESCRIBE full_table_name */ - { 321, -3 }, /* (280) cmd ::= RESET QUERY CACHE */ - { 321, -4 }, /* (281) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 387, 0 }, /* (282) analyze_opt ::= */ - { 387, -1 }, /* (283) analyze_opt ::= ANALYZE */ - { 388, 0 }, /* (284) explain_options ::= */ - { 388, -3 }, /* (285) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 388, -3 }, /* (286) explain_options ::= explain_options RATIO NK_FLOAT */ - { 321, -10 }, /* (287) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 321, -4 }, /* (288) cmd ::= DROP FUNCTION exists_opt function_name */ - { 389, 0 }, /* (289) agg_func_opt ::= */ - { 389, -1 }, /* (290) agg_func_opt ::= AGGREGATE */ - { 390, 0 }, /* (291) bufsize_opt ::= */ - { 390, -2 }, /* (292) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 321, -11 }, /* (293) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ - { 321, -4 }, /* (294) cmd ::= DROP STREAM exists_opt stream_name */ - { 384, 0 }, /* (295) stream_options ::= */ - { 384, -3 }, /* (296) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 384, -3 }, /* (297) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 384, -4 }, /* (298) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 384, -3 }, /* (299) stream_options ::= stream_options WATERMARK duration_literal */ - { 384, -4 }, /* (300) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 384, -3 }, /* (301) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 392, 0 }, /* (302) subtable_opt ::= */ - { 392, -4 }, /* (303) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 321, -3 }, /* (304) cmd ::= KILL CONNECTION NK_INTEGER */ - { 321, -3 }, /* (305) cmd ::= KILL QUERY NK_STRING */ - { 321, -3 }, /* (306) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 321, -2 }, /* (307) cmd ::= BALANCE VGROUP */ - { 321, -4 }, /* (308) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 321, -4 }, /* (309) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 321, -3 }, /* (310) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 394, -2 }, /* (311) dnode_list ::= DNODE NK_INTEGER */ - { 394, -3 }, /* (312) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 321, -4 }, /* (313) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 321, -1 }, /* (314) cmd ::= query_or_subquery */ - { 321, -7 }, /* (315) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 321, -4 }, /* (316) cmd ::= INSERT INTO full_table_name query_or_subquery */ - { 324, -1 }, /* (317) literal ::= NK_INTEGER */ - { 324, -1 }, /* (318) literal ::= NK_FLOAT */ - { 324, -1 }, /* (319) literal ::= NK_STRING */ - { 324, -1 }, /* (320) literal ::= NK_BOOL */ - { 324, -2 }, /* (321) literal ::= TIMESTAMP NK_STRING */ - { 324, -1 }, /* (322) literal ::= duration_literal */ - { 324, -1 }, /* (323) literal ::= NULL */ - { 324, -1 }, /* (324) literal ::= NK_QUESTION */ - { 368, -1 }, /* (325) duration_literal ::= NK_VARIABLE */ - { 396, -1 }, /* (326) signed ::= NK_INTEGER */ - { 396, -2 }, /* (327) signed ::= NK_PLUS NK_INTEGER */ - { 396, -2 }, /* (328) signed ::= NK_MINUS NK_INTEGER */ - { 396, -1 }, /* (329) signed ::= NK_FLOAT */ - { 396, -2 }, /* (330) signed ::= NK_PLUS NK_FLOAT */ - { 396, -2 }, /* (331) signed ::= NK_MINUS NK_FLOAT */ - { 357, -1 }, /* (332) signed_literal ::= signed */ - { 357, -1 }, /* (333) signed_literal ::= NK_STRING */ - { 357, -1 }, /* (334) signed_literal ::= NK_BOOL */ - { 357, -2 }, /* (335) signed_literal ::= TIMESTAMP NK_STRING */ - { 357, -1 }, /* (336) signed_literal ::= duration_literal */ - { 357, -1 }, /* (337) signed_literal ::= NULL */ - { 357, -1 }, /* (338) signed_literal ::= literal_func */ - { 357, -1 }, /* (339) signed_literal ::= NK_QUESTION */ - { 398, -1 }, /* (340) literal_list ::= signed_literal */ - { 398, -3 }, /* (341) literal_list ::= literal_list NK_COMMA signed_literal */ - { 332, -1 }, /* (342) db_name ::= NK_ID */ - { 363, -1 }, /* (343) table_name ::= NK_ID */ - { 355, -1 }, /* (344) column_name ::= NK_ID */ - { 370, -1 }, /* (345) function_name ::= NK_ID */ - { 399, -1 }, /* (346) table_alias ::= NK_ID */ - { 378, -1 }, /* (347) column_alias ::= NK_ID */ - { 326, -1 }, /* (348) user_name ::= NK_ID */ - { 333, -1 }, /* (349) topic_name ::= NK_ID */ - { 391, -1 }, /* (350) stream_name ::= NK_ID */ - { 386, -1 }, /* (351) cgroup_name ::= NK_ID */ - { 400, -1 }, /* (352) expr_or_subquery ::= expression */ - { 393, -1 }, /* (353) expression ::= literal */ - { 393, -1 }, /* (354) expression ::= pseudo_column */ - { 393, -1 }, /* (355) expression ::= column_reference */ - { 393, -1 }, /* (356) expression ::= function_expression */ - { 393, -1 }, /* (357) expression ::= case_when_expression */ - { 393, -3 }, /* (358) expression ::= NK_LP expression NK_RP */ - { 393, -2 }, /* (359) expression ::= NK_PLUS expr_or_subquery */ - { 393, -2 }, /* (360) expression ::= NK_MINUS expr_or_subquery */ - { 393, -3 }, /* (361) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 393, -3 }, /* (362) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 393, -3 }, /* (363) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 393, -3 }, /* (364) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 393, -3 }, /* (365) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 393, -3 }, /* (366) expression ::= column_reference NK_ARROW NK_STRING */ - { 393, -3 }, /* (367) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 393, -3 }, /* (368) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 360, -1 }, /* (369) expression_list ::= expr_or_subquery */ - { 360, -3 }, /* (370) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 402, -1 }, /* (371) column_reference ::= column_name */ - { 402, -3 }, /* (372) column_reference ::= table_name NK_DOT column_name */ - { 401, -1 }, /* (373) pseudo_column ::= ROWTS */ - { 401, -1 }, /* (374) pseudo_column ::= TBNAME */ - { 401, -3 }, /* (375) pseudo_column ::= table_name NK_DOT TBNAME */ - { 401, -1 }, /* (376) pseudo_column ::= QSTART */ - { 401, -1 }, /* (377) pseudo_column ::= QEND */ - { 401, -1 }, /* (378) pseudo_column ::= QDURATION */ - { 401, -1 }, /* (379) pseudo_column ::= WSTART */ - { 401, -1 }, /* (380) pseudo_column ::= WEND */ - { 401, -1 }, /* (381) pseudo_column ::= WDURATION */ - { 401, -1 }, /* (382) pseudo_column ::= IROWTS */ - { 401, -1 }, /* (383) pseudo_column ::= QTAGS */ - { 403, -4 }, /* (384) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 403, -4 }, /* (385) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 403, -6 }, /* (386) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 403, -1 }, /* (387) function_expression ::= literal_func */ - { 397, -3 }, /* (388) literal_func ::= noarg_func NK_LP NK_RP */ - { 397, -1 }, /* (389) literal_func ::= NOW */ - { 407, -1 }, /* (390) noarg_func ::= NOW */ - { 407, -1 }, /* (391) noarg_func ::= TODAY */ - { 407, -1 }, /* (392) noarg_func ::= TIMEZONE */ - { 407, -1 }, /* (393) noarg_func ::= DATABASE */ - { 407, -1 }, /* (394) noarg_func ::= CLIENT_VERSION */ - { 407, -1 }, /* (395) noarg_func ::= SERVER_VERSION */ - { 407, -1 }, /* (396) noarg_func ::= SERVER_STATUS */ - { 407, -1 }, /* (397) noarg_func ::= CURRENT_USER */ - { 407, -1 }, /* (398) noarg_func ::= USER */ - { 405, -1 }, /* (399) star_func ::= COUNT */ - { 405, -1 }, /* (400) star_func ::= FIRST */ - { 405, -1 }, /* (401) star_func ::= LAST */ - { 405, -1 }, /* (402) star_func ::= LAST_ROW */ - { 406, -1 }, /* (403) star_func_para_list ::= NK_STAR */ - { 406, -1 }, /* (404) star_func_para_list ::= other_para_list */ - { 408, -1 }, /* (405) other_para_list ::= star_func_para */ - { 408, -3 }, /* (406) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 409, -1 }, /* (407) star_func_para ::= expr_or_subquery */ - { 409, -3 }, /* (408) star_func_para ::= table_name NK_DOT NK_STAR */ - { 404, -4 }, /* (409) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 404, -5 }, /* (410) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 410, -1 }, /* (411) when_then_list ::= when_then_expr */ - { 410, -2 }, /* (412) when_then_list ::= when_then_list when_then_expr */ - { 413, -4 }, /* (413) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 411, 0 }, /* (414) case_when_else_opt ::= */ - { 411, -2 }, /* (415) case_when_else_opt ::= ELSE common_expression */ - { 414, -3 }, /* (416) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 414, -5 }, /* (417) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 414, -6 }, /* (418) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 414, -3 }, /* (419) predicate ::= expr_or_subquery IS NULL */ - { 414, -4 }, /* (420) predicate ::= expr_or_subquery IS NOT NULL */ - { 414, -3 }, /* (421) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 415, -1 }, /* (422) compare_op ::= NK_LT */ - { 415, -1 }, /* (423) compare_op ::= NK_GT */ - { 415, -1 }, /* (424) compare_op ::= NK_LE */ - { 415, -1 }, /* (425) compare_op ::= NK_GE */ - { 415, -1 }, /* (426) compare_op ::= NK_NE */ - { 415, -1 }, /* (427) compare_op ::= NK_EQ */ - { 415, -1 }, /* (428) compare_op ::= LIKE */ - { 415, -2 }, /* (429) compare_op ::= NOT LIKE */ - { 415, -1 }, /* (430) compare_op ::= MATCH */ - { 415, -1 }, /* (431) compare_op ::= NMATCH */ - { 415, -1 }, /* (432) compare_op ::= CONTAINS */ - { 416, -1 }, /* (433) in_op ::= IN */ - { 416, -2 }, /* (434) in_op ::= NOT IN */ - { 417, -3 }, /* (435) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 418, -1 }, /* (436) boolean_value_expression ::= boolean_primary */ - { 418, -2 }, /* (437) boolean_value_expression ::= NOT boolean_primary */ - { 418, -3 }, /* (438) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 418, -3 }, /* (439) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 419, -1 }, /* (440) boolean_primary ::= predicate */ - { 419, -3 }, /* (441) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 412, -1 }, /* (442) common_expression ::= expr_or_subquery */ - { 412, -1 }, /* (443) common_expression ::= boolean_value_expression */ - { 420, 0 }, /* (444) from_clause_opt ::= */ - { 420, -2 }, /* (445) from_clause_opt ::= FROM table_reference_list */ - { 421, -1 }, /* (446) table_reference_list ::= table_reference */ - { 421, -3 }, /* (447) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 422, -1 }, /* (448) table_reference ::= table_primary */ - { 422, -1 }, /* (449) table_reference ::= joined_table */ - { 423, -2 }, /* (450) table_primary ::= table_name alias_opt */ - { 423, -4 }, /* (451) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 423, -2 }, /* (452) table_primary ::= subquery alias_opt */ - { 423, -1 }, /* (453) table_primary ::= parenthesized_joined_table */ - { 425, 0 }, /* (454) alias_opt ::= */ - { 425, -1 }, /* (455) alias_opt ::= table_alias */ - { 425, -2 }, /* (456) alias_opt ::= AS table_alias */ - { 427, -3 }, /* (457) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 427, -3 }, /* (458) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 424, -6 }, /* (459) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 428, 0 }, /* (460) join_type ::= */ - { 428, -1 }, /* (461) join_type ::= INNER */ - { 430, -12 }, /* (462) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 431, 0 }, /* (463) set_quantifier_opt ::= */ - { 431, -1 }, /* (464) set_quantifier_opt ::= DISTINCT */ - { 431, -1 }, /* (465) set_quantifier_opt ::= ALL */ - { 432, -1 }, /* (466) select_list ::= select_item */ - { 432, -3 }, /* (467) select_list ::= select_list NK_COMMA select_item */ - { 440, -1 }, /* (468) select_item ::= NK_STAR */ - { 440, -1 }, /* (469) select_item ::= common_expression */ - { 440, -2 }, /* (470) select_item ::= common_expression column_alias */ - { 440, -3 }, /* (471) select_item ::= common_expression AS column_alias */ - { 440, -3 }, /* (472) select_item ::= table_name NK_DOT NK_STAR */ - { 395, 0 }, /* (473) where_clause_opt ::= */ - { 395, -2 }, /* (474) where_clause_opt ::= WHERE search_condition */ - { 433, 0 }, /* (475) partition_by_clause_opt ::= */ - { 433, -3 }, /* (476) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 441, -1 }, /* (477) partition_list ::= partition_item */ - { 441, -3 }, /* (478) partition_list ::= partition_list NK_COMMA partition_item */ - { 442, -1 }, /* (479) partition_item ::= expr_or_subquery */ - { 442, -2 }, /* (480) partition_item ::= expr_or_subquery column_alias */ - { 442, -3 }, /* (481) partition_item ::= expr_or_subquery AS column_alias */ - { 437, 0 }, /* (482) twindow_clause_opt ::= */ - { 437, -6 }, /* (483) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 437, -4 }, /* (484) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 437, -6 }, /* (485) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 437, -8 }, /* (486) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 381, 0 }, /* (487) sliding_opt ::= */ - { 381, -4 }, /* (488) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 436, 0 }, /* (489) fill_opt ::= */ - { 436, -4 }, /* (490) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 436, -6 }, /* (491) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 443, -1 }, /* (492) fill_mode ::= NONE */ - { 443, -1 }, /* (493) fill_mode ::= PREV */ - { 443, -1 }, /* (494) fill_mode ::= NULL */ - { 443, -1 }, /* (495) fill_mode ::= LINEAR */ - { 443, -1 }, /* (496) fill_mode ::= NEXT */ - { 438, 0 }, /* (497) group_by_clause_opt ::= */ - { 438, -3 }, /* (498) group_by_clause_opt ::= GROUP BY group_by_list */ - { 444, -1 }, /* (499) group_by_list ::= expr_or_subquery */ - { 444, -3 }, /* (500) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 439, 0 }, /* (501) having_clause_opt ::= */ - { 439, -2 }, /* (502) having_clause_opt ::= HAVING search_condition */ - { 434, 0 }, /* (503) range_opt ::= */ - { 434, -6 }, /* (504) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 435, 0 }, /* (505) every_opt ::= */ - { 435, -4 }, /* (506) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 445, -4 }, /* (507) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 446, -1 }, /* (508) query_simple ::= query_specification */ - { 446, -1 }, /* (509) query_simple ::= union_query_expression */ - { 450, -4 }, /* (510) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 450, -3 }, /* (511) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 451, -1 }, /* (512) query_simple_or_subquery ::= query_simple */ - { 451, -1 }, /* (513) query_simple_or_subquery ::= subquery */ - { 385, -1 }, /* (514) query_or_subquery ::= query_expression */ - { 385, -1 }, /* (515) query_or_subquery ::= subquery */ - { 447, 0 }, /* (516) order_by_clause_opt ::= */ - { 447, -3 }, /* (517) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 448, 0 }, /* (518) slimit_clause_opt ::= */ - { 448, -2 }, /* (519) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 448, -4 }, /* (520) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 448, -4 }, /* (521) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 449, 0 }, /* (522) limit_clause_opt ::= */ - { 449, -2 }, /* (523) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 449, -4 }, /* (524) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 449, -4 }, /* (525) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 426, -3 }, /* (526) subquery ::= NK_LP query_expression NK_RP */ - { 426, -3 }, /* (527) subquery ::= NK_LP subquery NK_RP */ - { 429, -1 }, /* (528) search_condition ::= common_expression */ - { 452, -1 }, /* (529) sort_specification_list ::= sort_specification */ - { 452, -3 }, /* (530) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 453, -3 }, /* (531) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 454, 0 }, /* (532) ordering_specification_opt ::= */ - { 454, -1 }, /* (533) ordering_specification_opt ::= ASC */ - { 454, -1 }, /* (534) ordering_specification_opt ::= DESC */ - { 455, 0 }, /* (535) null_ordering_opt ::= */ - { 455, -2 }, /* (536) null_ordering_opt ::= NULLS FIRST */ - { 455, -2 }, /* (537) null_ordering_opt ::= NULLS LAST */ + { 324, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 324, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 325, 0 }, /* (2) account_options ::= */ + { 325, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 325, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 325, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 325, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 325, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 325, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 325, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 325, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 325, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 326, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 326, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 328, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 328, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 328, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 328, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 328, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 328, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 328, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 328, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 328, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 328, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 324, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 324, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 324, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 324, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 324, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 330, 0 }, /* (29) sysinfo_opt ::= */ + { 330, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 324, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ + { 324, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 331, -1 }, /* (33) privileges ::= ALL */ + { 331, -1 }, /* (34) privileges ::= priv_type_list */ + { 331, -1 }, /* (35) privileges ::= SUBSCRIBE */ + { 333, -1 }, /* (36) priv_type_list ::= priv_type */ + { 333, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 334, -1 }, /* (38) priv_type ::= READ */ + { 334, -1 }, /* (39) priv_type ::= WRITE */ + { 332, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 332, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + { 332, -1 }, /* (42) priv_level ::= topic_name */ + { 324, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ + { 324, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 324, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ + { 324, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ + { 324, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 324, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 324, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ + { 324, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 337, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ + { 337, -1 }, /* (52) dnode_endpoint ::= NK_ID */ + { 337, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ + { 338, 0 }, /* (54) force_opt ::= */ + { 338, -1 }, /* (55) force_opt ::= FORCE */ + { 324, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ + { 324, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 324, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 324, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 324, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ + { 324, -2 }, /* (68) cmd ::= USE db_name */ + { 324, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 324, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ + { 324, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ + { 339, -3 }, /* (72) not_exists_opt ::= IF NOT EXISTS */ + { 339, 0 }, /* (73) not_exists_opt ::= */ + { 341, -2 }, /* (74) exists_opt ::= IF EXISTS */ + { 341, 0 }, /* (75) exists_opt ::= */ + { 340, 0 }, /* (76) db_options ::= */ + { 340, -3 }, /* (77) db_options ::= db_options BUFFER NK_INTEGER */ + { 340, -3 }, /* (78) db_options ::= db_options CACHEMODEL NK_STRING */ + { 340, -3 }, /* (79) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 340, -3 }, /* (80) db_options ::= db_options COMP NK_INTEGER */ + { 340, -3 }, /* (81) db_options ::= db_options DURATION NK_INTEGER */ + { 340, -3 }, /* (82) db_options ::= db_options DURATION NK_VARIABLE */ + { 340, -3 }, /* (83) db_options ::= db_options MAXROWS NK_INTEGER */ + { 340, -3 }, /* (84) db_options ::= db_options MINROWS NK_INTEGER */ + { 340, -3 }, /* (85) db_options ::= db_options KEEP integer_list */ + { 340, -3 }, /* (86) db_options ::= db_options KEEP variable_list */ + { 340, -3 }, /* (87) db_options ::= db_options PAGES NK_INTEGER */ + { 340, -3 }, /* (88) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 340, -3 }, /* (89) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 340, -3 }, /* (90) db_options ::= db_options PRECISION NK_STRING */ + { 340, -3 }, /* (91) db_options ::= db_options REPLICA NK_INTEGER */ + { 340, -3 }, /* (92) db_options ::= db_options VGROUPS NK_INTEGER */ + { 340, -3 }, /* (93) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 340, -3 }, /* (94) db_options ::= db_options RETENTIONS retention_list */ + { 340, -3 }, /* (95) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 340, -3 }, /* (96) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 340, -3 }, /* (97) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 340, -3 }, /* (98) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 340, -4 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 340, -3 }, /* (100) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 340, -4 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 340, -3 }, /* (102) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 340, -3 }, /* (103) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 340, -3 }, /* (104) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 340, -3 }, /* (105) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 340, -3 }, /* (106) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 342, -1 }, /* (107) alter_db_options ::= alter_db_option */ + { 342, -2 }, /* (108) alter_db_options ::= alter_db_options alter_db_option */ + { 347, -2 }, /* (109) alter_db_option ::= BUFFER NK_INTEGER */ + { 347, -2 }, /* (110) alter_db_option ::= CACHEMODEL NK_STRING */ + { 347, -2 }, /* (111) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 347, -2 }, /* (112) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 347, -2 }, /* (113) alter_db_option ::= KEEP integer_list */ + { 347, -2 }, /* (114) alter_db_option ::= KEEP variable_list */ + { 347, -2 }, /* (115) alter_db_option ::= PAGES NK_INTEGER */ + { 347, -2 }, /* (116) alter_db_option ::= REPLICA NK_INTEGER */ + { 347, -2 }, /* (117) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 347, -2 }, /* (118) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 344, -1 }, /* (119) integer_list ::= NK_INTEGER */ + { 344, -3 }, /* (120) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 345, -1 }, /* (121) variable_list ::= NK_VARIABLE */ + { 345, -3 }, /* (122) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 346, -1 }, /* (123) retention_list ::= retention */ + { 346, -3 }, /* (124) retention_list ::= retention_list NK_COMMA retention */ + { 348, -3 }, /* (125) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 343, 0 }, /* (126) speed_opt ::= */ + { 343, -2 }, /* (127) speed_opt ::= MAX_SPEED NK_INTEGER */ + { 324, -9 }, /* (128) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 324, -3 }, /* (129) cmd ::= CREATE TABLE multi_create_clause */ + { 324, -9 }, /* (130) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 324, -3 }, /* (131) cmd ::= DROP TABLE multi_drop_clause */ + { 324, -4 }, /* (132) cmd ::= DROP STABLE exists_opt full_table_name */ + { 324, -3 }, /* (133) cmd ::= ALTER TABLE alter_table_clause */ + { 324, -3 }, /* (134) cmd ::= ALTER STABLE alter_table_clause */ + { 356, -2 }, /* (135) alter_table_clause ::= full_table_name alter_table_options */ + { 356, -5 }, /* (136) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 356, -4 }, /* (137) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 356, -5 }, /* (138) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 356, -5 }, /* (139) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 356, -5 }, /* (140) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 356, -4 }, /* (141) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 356, -5 }, /* (142) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 356, -5 }, /* (143) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 356, -6 }, /* (144) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 353, -1 }, /* (145) multi_create_clause ::= create_subtable_clause */ + { 353, -2 }, /* (146) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 361, -10 }, /* (147) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 355, -1 }, /* (148) multi_drop_clause ::= drop_table_clause */ + { 355, -2 }, /* (149) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 364, -2 }, /* (150) drop_table_clause ::= exists_opt full_table_name */ + { 362, 0 }, /* (151) specific_cols_opt ::= */ + { 362, -3 }, /* (152) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 349, -1 }, /* (153) full_table_name ::= table_name */ + { 349, -3 }, /* (154) full_table_name ::= db_name NK_DOT table_name */ + { 350, -1 }, /* (155) column_def_list ::= column_def */ + { 350, -3 }, /* (156) column_def_list ::= column_def_list NK_COMMA column_def */ + { 367, -2 }, /* (157) column_def ::= column_name type_name */ + { 367, -4 }, /* (158) column_def ::= column_name type_name COMMENT NK_STRING */ + { 359, -1 }, /* (159) type_name ::= BOOL */ + { 359, -1 }, /* (160) type_name ::= TINYINT */ + { 359, -1 }, /* (161) type_name ::= SMALLINT */ + { 359, -1 }, /* (162) type_name ::= INT */ + { 359, -1 }, /* (163) type_name ::= INTEGER */ + { 359, -1 }, /* (164) type_name ::= BIGINT */ + { 359, -1 }, /* (165) type_name ::= FLOAT */ + { 359, -1 }, /* (166) type_name ::= DOUBLE */ + { 359, -4 }, /* (167) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 359, -1 }, /* (168) type_name ::= TIMESTAMP */ + { 359, -4 }, /* (169) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 359, -2 }, /* (170) type_name ::= TINYINT UNSIGNED */ + { 359, -2 }, /* (171) type_name ::= SMALLINT UNSIGNED */ + { 359, -2 }, /* (172) type_name ::= INT UNSIGNED */ + { 359, -2 }, /* (173) type_name ::= BIGINT UNSIGNED */ + { 359, -1 }, /* (174) type_name ::= JSON */ + { 359, -4 }, /* (175) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 359, -1 }, /* (176) type_name ::= MEDIUMBLOB */ + { 359, -1 }, /* (177) type_name ::= BLOB */ + { 359, -4 }, /* (178) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 359, -1 }, /* (179) type_name ::= DECIMAL */ + { 359, -4 }, /* (180) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 359, -6 }, /* (181) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 351, 0 }, /* (182) tags_def_opt ::= */ + { 351, -1 }, /* (183) tags_def_opt ::= tags_def */ + { 354, -4 }, /* (184) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 352, 0 }, /* (185) table_options ::= */ + { 352, -3 }, /* (186) table_options ::= table_options COMMENT NK_STRING */ + { 352, -3 }, /* (187) table_options ::= table_options MAX_DELAY duration_list */ + { 352, -3 }, /* (188) table_options ::= table_options WATERMARK duration_list */ + { 352, -5 }, /* (189) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 352, -3 }, /* (190) table_options ::= table_options TTL NK_INTEGER */ + { 352, -5 }, /* (191) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 352, -3 }, /* (192) table_options ::= table_options DELETE_MARK duration_list */ + { 357, -1 }, /* (193) alter_table_options ::= alter_table_option */ + { 357, -2 }, /* (194) alter_table_options ::= alter_table_options alter_table_option */ + { 370, -2 }, /* (195) alter_table_option ::= COMMENT NK_STRING */ + { 370, -2 }, /* (196) alter_table_option ::= TTL NK_INTEGER */ + { 368, -1 }, /* (197) duration_list ::= duration_literal */ + { 368, -3 }, /* (198) duration_list ::= duration_list NK_COMMA duration_literal */ + { 369, -1 }, /* (199) rollup_func_list ::= rollup_func_name */ + { 369, -3 }, /* (200) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 372, -1 }, /* (201) rollup_func_name ::= function_name */ + { 372, -1 }, /* (202) rollup_func_name ::= FIRST */ + { 372, -1 }, /* (203) rollup_func_name ::= LAST */ + { 365, -1 }, /* (204) col_name_list ::= col_name */ + { 365, -3 }, /* (205) col_name_list ::= col_name_list NK_COMMA col_name */ + { 374, -1 }, /* (206) col_name ::= column_name */ + { 324, -2 }, /* (207) cmd ::= SHOW DNODES */ + { 324, -2 }, /* (208) cmd ::= SHOW USERS */ + { 324, -3 }, /* (209) cmd ::= SHOW USER PRIVILEGES */ + { 324, -2 }, /* (210) cmd ::= SHOW DATABASES */ + { 324, -4 }, /* (211) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 324, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 324, -3 }, /* (213) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 324, -2 }, /* (214) cmd ::= SHOW MNODES */ + { 324, -2 }, /* (215) cmd ::= SHOW QNODES */ + { 324, -2 }, /* (216) cmd ::= SHOW FUNCTIONS */ + { 324, -5 }, /* (217) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 324, -2 }, /* (218) cmd ::= SHOW STREAMS */ + { 324, -2 }, /* (219) cmd ::= SHOW ACCOUNTS */ + { 324, -2 }, /* (220) cmd ::= SHOW APPS */ + { 324, -2 }, /* (221) cmd ::= SHOW CONNECTIONS */ + { 324, -2 }, /* (222) cmd ::= SHOW LICENCES */ + { 324, -2 }, /* (223) cmd ::= SHOW GRANTS */ + { 324, -4 }, /* (224) cmd ::= SHOW CREATE DATABASE db_name */ + { 324, -4 }, /* (225) cmd ::= SHOW CREATE TABLE full_table_name */ + { 324, -4 }, /* (226) cmd ::= SHOW CREATE STABLE full_table_name */ + { 324, -2 }, /* (227) cmd ::= SHOW QUERIES */ + { 324, -2 }, /* (228) cmd ::= SHOW SCORES */ + { 324, -2 }, /* (229) cmd ::= SHOW TOPICS */ + { 324, -2 }, /* (230) cmd ::= SHOW VARIABLES */ + { 324, -3 }, /* (231) cmd ::= SHOW CLUSTER VARIABLES */ + { 324, -3 }, /* (232) cmd ::= SHOW LOCAL VARIABLES */ + { 324, -5 }, /* (233) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 324, -2 }, /* (234) cmd ::= SHOW BNODES */ + { 324, -2 }, /* (235) cmd ::= SHOW SNODES */ + { 324, -2 }, /* (236) cmd ::= SHOW CLUSTER */ + { 324, -2 }, /* (237) cmd ::= SHOW TRANSACTIONS */ + { 324, -4 }, /* (238) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 324, -2 }, /* (239) cmd ::= SHOW CONSUMERS */ + { 324, -2 }, /* (240) cmd ::= SHOW SUBSCRIPTIONS */ + { 324, -5 }, /* (241) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 324, -7 }, /* (242) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 324, -3 }, /* (243) cmd ::= SHOW VNODES NK_INTEGER */ + { 324, -3 }, /* (244) cmd ::= SHOW VNODES NK_STRING */ + { 375, 0 }, /* (245) db_name_cond_opt ::= */ + { 375, -2 }, /* (246) db_name_cond_opt ::= db_name NK_DOT */ + { 376, 0 }, /* (247) like_pattern_opt ::= */ + { 376, -2 }, /* (248) like_pattern_opt ::= LIKE NK_STRING */ + { 377, -1 }, /* (249) table_name_cond ::= table_name */ + { 378, 0 }, /* (250) from_db_opt ::= */ + { 378, -2 }, /* (251) from_db_opt ::= FROM db_name */ + { 379, 0 }, /* (252) tag_list_opt ::= */ + { 379, -1 }, /* (253) tag_list_opt ::= tag_item */ + { 379, -3 }, /* (254) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 380, -1 }, /* (255) tag_item ::= TBNAME */ + { 380, -1 }, /* (256) tag_item ::= QTAGS */ + { 380, -1 }, /* (257) tag_item ::= column_name */ + { 380, -2 }, /* (258) tag_item ::= column_name column_alias */ + { 380, -3 }, /* (259) tag_item ::= column_name AS column_alias */ + { 324, -8 }, /* (260) cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ + { 324, -4 }, /* (261) cmd ::= DROP INDEX exists_opt full_table_name */ + { 382, -10 }, /* (262) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 382, -12 }, /* (263) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 383, -1 }, /* (264) func_list ::= func */ + { 383, -3 }, /* (265) func_list ::= func_list NK_COMMA func */ + { 386, -4 }, /* (266) func ::= function_name NK_LP expression_list NK_RP */ + { 385, 0 }, /* (267) sma_stream_opt ::= */ + { 385, -3 }, /* (268) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 385, -3 }, /* (269) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 385, -3 }, /* (270) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 324, -6 }, /* (271) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 324, -7 }, /* (272) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 324, -9 }, /* (273) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 324, -7 }, /* (274) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 324, -9 }, /* (275) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 324, -4 }, /* (276) cmd ::= DROP TOPIC exists_opt topic_name */ + { 324, -7 }, /* (277) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 324, -2 }, /* (278) cmd ::= DESC full_table_name */ + { 324, -2 }, /* (279) cmd ::= DESCRIBE full_table_name */ + { 324, -3 }, /* (280) cmd ::= RESET QUERY CACHE */ + { 324, -4 }, /* (281) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 389, 0 }, /* (282) analyze_opt ::= */ + { 389, -1 }, /* (283) analyze_opt ::= ANALYZE */ + { 390, 0 }, /* (284) explain_options ::= */ + { 390, -3 }, /* (285) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 390, -3 }, /* (286) explain_options ::= explain_options RATIO NK_FLOAT */ + { 324, -10 }, /* (287) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 324, -4 }, /* (288) cmd ::= DROP FUNCTION exists_opt function_name */ + { 391, 0 }, /* (289) agg_func_opt ::= */ + { 391, -1 }, /* (290) agg_func_opt ::= AGGREGATE */ + { 392, 0 }, /* (291) bufsize_opt ::= */ + { 392, -2 }, /* (292) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 324, -11 }, /* (293) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ + { 324, -4 }, /* (294) cmd ::= DROP STREAM exists_opt stream_name */ + { 394, 0 }, /* (295) stream_options ::= */ + { 394, -3 }, /* (296) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 394, -3 }, /* (297) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 394, -4 }, /* (298) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 394, -3 }, /* (299) stream_options ::= stream_options WATERMARK duration_literal */ + { 394, -4 }, /* (300) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 394, -3 }, /* (301) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 395, 0 }, /* (302) subtable_opt ::= */ + { 395, -4 }, /* (303) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 324, -3 }, /* (304) cmd ::= KILL CONNECTION NK_INTEGER */ + { 324, -3 }, /* (305) cmd ::= KILL QUERY NK_STRING */ + { 324, -3 }, /* (306) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 324, -2 }, /* (307) cmd ::= BALANCE VGROUP */ + { 324, -4 }, /* (308) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 324, -4 }, /* (309) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 324, -3 }, /* (310) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 397, -2 }, /* (311) dnode_list ::= DNODE NK_INTEGER */ + { 397, -3 }, /* (312) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 324, -4 }, /* (313) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 324, -1 }, /* (314) cmd ::= query_or_subquery */ + { 324, -7 }, /* (315) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 324, -4 }, /* (316) cmd ::= INSERT INTO full_table_name query_or_subquery */ + { 327, -1 }, /* (317) literal ::= NK_INTEGER */ + { 327, -1 }, /* (318) literal ::= NK_FLOAT */ + { 327, -1 }, /* (319) literal ::= NK_STRING */ + { 327, -1 }, /* (320) literal ::= NK_BOOL */ + { 327, -2 }, /* (321) literal ::= TIMESTAMP NK_STRING */ + { 327, -1 }, /* (322) literal ::= duration_literal */ + { 327, -1 }, /* (323) literal ::= NULL */ + { 327, -1 }, /* (324) literal ::= NK_QUESTION */ + { 371, -1 }, /* (325) duration_literal ::= NK_VARIABLE */ + { 399, -1 }, /* (326) signed ::= NK_INTEGER */ + { 399, -2 }, /* (327) signed ::= NK_PLUS NK_INTEGER */ + { 399, -2 }, /* (328) signed ::= NK_MINUS NK_INTEGER */ + { 399, -1 }, /* (329) signed ::= NK_FLOAT */ + { 399, -2 }, /* (330) signed ::= NK_PLUS NK_FLOAT */ + { 399, -2 }, /* (331) signed ::= NK_MINUS NK_FLOAT */ + { 360, -1 }, /* (332) signed_literal ::= signed */ + { 360, -1 }, /* (333) signed_literal ::= NK_STRING */ + { 360, -1 }, /* (334) signed_literal ::= NK_BOOL */ + { 360, -2 }, /* (335) signed_literal ::= TIMESTAMP NK_STRING */ + { 360, -1 }, /* (336) signed_literal ::= duration_literal */ + { 360, -1 }, /* (337) signed_literal ::= NULL */ + { 360, -1 }, /* (338) signed_literal ::= literal_func */ + { 360, -1 }, /* (339) signed_literal ::= NK_QUESTION */ + { 401, -1 }, /* (340) literal_list ::= signed_literal */ + { 401, -3 }, /* (341) literal_list ::= literal_list NK_COMMA signed_literal */ + { 335, -1 }, /* (342) db_name ::= NK_ID */ + { 366, -1 }, /* (343) table_name ::= NK_ID */ + { 358, -1 }, /* (344) column_name ::= NK_ID */ + { 373, -1 }, /* (345) function_name ::= NK_ID */ + { 402, -1 }, /* (346) table_alias ::= NK_ID */ + { 381, -1 }, /* (347) column_alias ::= NK_ID */ + { 329, -1 }, /* (348) user_name ::= NK_ID */ + { 336, -1 }, /* (349) topic_name ::= NK_ID */ + { 393, -1 }, /* (350) stream_name ::= NK_ID */ + { 388, -1 }, /* (351) cgroup_name ::= NK_ID */ + { 403, -1 }, /* (352) expr_or_subquery ::= expression */ + { 396, -1 }, /* (353) expression ::= literal */ + { 396, -1 }, /* (354) expression ::= pseudo_column */ + { 396, -1 }, /* (355) expression ::= column_reference */ + { 396, -1 }, /* (356) expression ::= function_expression */ + { 396, -1 }, /* (357) expression ::= case_when_expression */ + { 396, -3 }, /* (358) expression ::= NK_LP expression NK_RP */ + { 396, -2 }, /* (359) expression ::= NK_PLUS expr_or_subquery */ + { 396, -2 }, /* (360) expression ::= NK_MINUS expr_or_subquery */ + { 396, -3 }, /* (361) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 396, -3 }, /* (362) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 396, -3 }, /* (363) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 396, -3 }, /* (364) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 396, -3 }, /* (365) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 396, -3 }, /* (366) expression ::= column_reference NK_ARROW NK_STRING */ + { 396, -3 }, /* (367) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 396, -3 }, /* (368) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 363, -1 }, /* (369) expression_list ::= expr_or_subquery */ + { 363, -3 }, /* (370) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 405, -1 }, /* (371) column_reference ::= column_name */ + { 405, -3 }, /* (372) column_reference ::= table_name NK_DOT column_name */ + { 404, -1 }, /* (373) pseudo_column ::= ROWTS */ + { 404, -1 }, /* (374) pseudo_column ::= TBNAME */ + { 404, -3 }, /* (375) pseudo_column ::= table_name NK_DOT TBNAME */ + { 404, -1 }, /* (376) pseudo_column ::= QSTART */ + { 404, -1 }, /* (377) pseudo_column ::= QEND */ + { 404, -1 }, /* (378) pseudo_column ::= QDURATION */ + { 404, -1 }, /* (379) pseudo_column ::= WSTART */ + { 404, -1 }, /* (380) pseudo_column ::= WEND */ + { 404, -1 }, /* (381) pseudo_column ::= WDURATION */ + { 404, -1 }, /* (382) pseudo_column ::= IROWTS */ + { 404, -1 }, /* (383) pseudo_column ::= QTAGS */ + { 406, -4 }, /* (384) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 406, -4 }, /* (385) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 406, -6 }, /* (386) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 406, -1 }, /* (387) function_expression ::= literal_func */ + { 400, -3 }, /* (388) literal_func ::= noarg_func NK_LP NK_RP */ + { 400, -1 }, /* (389) literal_func ::= NOW */ + { 410, -1 }, /* (390) noarg_func ::= NOW */ + { 410, -1 }, /* (391) noarg_func ::= TODAY */ + { 410, -1 }, /* (392) noarg_func ::= TIMEZONE */ + { 410, -1 }, /* (393) noarg_func ::= DATABASE */ + { 410, -1 }, /* (394) noarg_func ::= CLIENT_VERSION */ + { 410, -1 }, /* (395) noarg_func ::= SERVER_VERSION */ + { 410, -1 }, /* (396) noarg_func ::= SERVER_STATUS */ + { 410, -1 }, /* (397) noarg_func ::= CURRENT_USER */ + { 410, -1 }, /* (398) noarg_func ::= USER */ + { 408, -1 }, /* (399) star_func ::= COUNT */ + { 408, -1 }, /* (400) star_func ::= FIRST */ + { 408, -1 }, /* (401) star_func ::= LAST */ + { 408, -1 }, /* (402) star_func ::= LAST_ROW */ + { 409, -1 }, /* (403) star_func_para_list ::= NK_STAR */ + { 409, -1 }, /* (404) star_func_para_list ::= other_para_list */ + { 411, -1 }, /* (405) other_para_list ::= star_func_para */ + { 411, -3 }, /* (406) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 412, -1 }, /* (407) star_func_para ::= expr_or_subquery */ + { 412, -3 }, /* (408) star_func_para ::= table_name NK_DOT NK_STAR */ + { 407, -4 }, /* (409) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 407, -5 }, /* (410) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 413, -1 }, /* (411) when_then_list ::= when_then_expr */ + { 413, -2 }, /* (412) when_then_list ::= when_then_list when_then_expr */ + { 416, -4 }, /* (413) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 414, 0 }, /* (414) case_when_else_opt ::= */ + { 414, -2 }, /* (415) case_when_else_opt ::= ELSE common_expression */ + { 417, -3 }, /* (416) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 417, -5 }, /* (417) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 417, -6 }, /* (418) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 417, -3 }, /* (419) predicate ::= expr_or_subquery IS NULL */ + { 417, -4 }, /* (420) predicate ::= expr_or_subquery IS NOT NULL */ + { 417, -3 }, /* (421) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 418, -1 }, /* (422) compare_op ::= NK_LT */ + { 418, -1 }, /* (423) compare_op ::= NK_GT */ + { 418, -1 }, /* (424) compare_op ::= NK_LE */ + { 418, -1 }, /* (425) compare_op ::= NK_GE */ + { 418, -1 }, /* (426) compare_op ::= NK_NE */ + { 418, -1 }, /* (427) compare_op ::= NK_EQ */ + { 418, -1 }, /* (428) compare_op ::= LIKE */ + { 418, -2 }, /* (429) compare_op ::= NOT LIKE */ + { 418, -1 }, /* (430) compare_op ::= MATCH */ + { 418, -1 }, /* (431) compare_op ::= NMATCH */ + { 418, -1 }, /* (432) compare_op ::= CONTAINS */ + { 419, -1 }, /* (433) in_op ::= IN */ + { 419, -2 }, /* (434) in_op ::= NOT IN */ + { 420, -3 }, /* (435) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 421, -1 }, /* (436) boolean_value_expression ::= boolean_primary */ + { 421, -2 }, /* (437) boolean_value_expression ::= NOT boolean_primary */ + { 421, -3 }, /* (438) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 421, -3 }, /* (439) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 422, -1 }, /* (440) boolean_primary ::= predicate */ + { 422, -3 }, /* (441) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 415, -1 }, /* (442) common_expression ::= expr_or_subquery */ + { 415, -1 }, /* (443) common_expression ::= boolean_value_expression */ + { 423, 0 }, /* (444) from_clause_opt ::= */ + { 423, -2 }, /* (445) from_clause_opt ::= FROM table_reference_list */ + { 424, -1 }, /* (446) table_reference_list ::= table_reference */ + { 424, -3 }, /* (447) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 425, -1 }, /* (448) table_reference ::= table_primary */ + { 425, -1 }, /* (449) table_reference ::= joined_table */ + { 426, -2 }, /* (450) table_primary ::= table_name alias_opt */ + { 426, -4 }, /* (451) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 426, -2 }, /* (452) table_primary ::= subquery alias_opt */ + { 426, -1 }, /* (453) table_primary ::= parenthesized_joined_table */ + { 428, 0 }, /* (454) alias_opt ::= */ + { 428, -1 }, /* (455) alias_opt ::= table_alias */ + { 428, -2 }, /* (456) alias_opt ::= AS table_alias */ + { 430, -3 }, /* (457) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 430, -3 }, /* (458) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 427, -6 }, /* (459) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 431, 0 }, /* (460) join_type ::= */ + { 431, -1 }, /* (461) join_type ::= INNER */ + { 433, -12 }, /* (462) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 434, 0 }, /* (463) set_quantifier_opt ::= */ + { 434, -1 }, /* (464) set_quantifier_opt ::= DISTINCT */ + { 434, -1 }, /* (465) set_quantifier_opt ::= ALL */ + { 435, -1 }, /* (466) select_list ::= select_item */ + { 435, -3 }, /* (467) select_list ::= select_list NK_COMMA select_item */ + { 443, -1 }, /* (468) select_item ::= NK_STAR */ + { 443, -1 }, /* (469) select_item ::= common_expression */ + { 443, -2 }, /* (470) select_item ::= common_expression column_alias */ + { 443, -3 }, /* (471) select_item ::= common_expression AS column_alias */ + { 443, -3 }, /* (472) select_item ::= table_name NK_DOT NK_STAR */ + { 398, 0 }, /* (473) where_clause_opt ::= */ + { 398, -2 }, /* (474) where_clause_opt ::= WHERE search_condition */ + { 436, 0 }, /* (475) partition_by_clause_opt ::= */ + { 436, -3 }, /* (476) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 444, -1 }, /* (477) partition_list ::= partition_item */ + { 444, -3 }, /* (478) partition_list ::= partition_list NK_COMMA partition_item */ + { 445, -1 }, /* (479) partition_item ::= expr_or_subquery */ + { 445, -2 }, /* (480) partition_item ::= expr_or_subquery column_alias */ + { 445, -3 }, /* (481) partition_item ::= expr_or_subquery AS column_alias */ + { 440, 0 }, /* (482) twindow_clause_opt ::= */ + { 440, -6 }, /* (483) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 440, -4 }, /* (484) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 440, -6 }, /* (485) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 440, -8 }, /* (486) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 440, -7 }, /* (487) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 384, 0 }, /* (488) sliding_opt ::= */ + { 384, -4 }, /* (489) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 439, 0 }, /* (490) fill_opt ::= */ + { 439, -4 }, /* (491) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 439, -6 }, /* (492) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 446, -1 }, /* (493) fill_mode ::= NONE */ + { 446, -1 }, /* (494) fill_mode ::= PREV */ + { 446, -1 }, /* (495) fill_mode ::= NULL */ + { 446, -1 }, /* (496) fill_mode ::= LINEAR */ + { 446, -1 }, /* (497) fill_mode ::= NEXT */ + { 441, 0 }, /* (498) group_by_clause_opt ::= */ + { 441, -3 }, /* (499) group_by_clause_opt ::= GROUP BY group_by_list */ + { 447, -1 }, /* (500) group_by_list ::= expr_or_subquery */ + { 447, -3 }, /* (501) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 442, 0 }, /* (502) having_clause_opt ::= */ + { 442, -2 }, /* (503) having_clause_opt ::= HAVING search_condition */ + { 437, 0 }, /* (504) range_opt ::= */ + { 437, -6 }, /* (505) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 438, 0 }, /* (506) every_opt ::= */ + { 438, -4 }, /* (507) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 448, -4 }, /* (508) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 449, -1 }, /* (509) query_simple ::= query_specification */ + { 449, -1 }, /* (510) query_simple ::= union_query_expression */ + { 453, -4 }, /* (511) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 453, -3 }, /* (512) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 454, -1 }, /* (513) query_simple_or_subquery ::= query_simple */ + { 454, -1 }, /* (514) query_simple_or_subquery ::= subquery */ + { 387, -1 }, /* (515) query_or_subquery ::= query_expression */ + { 387, -1 }, /* (516) query_or_subquery ::= subquery */ + { 450, 0 }, /* (517) order_by_clause_opt ::= */ + { 450, -3 }, /* (518) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 451, 0 }, /* (519) slimit_clause_opt ::= */ + { 451, -2 }, /* (520) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 451, -4 }, /* (521) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 451, -4 }, /* (522) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 452, 0 }, /* (523) limit_clause_opt ::= */ + { 452, -2 }, /* (524) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 452, -4 }, /* (525) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 452, -4 }, /* (526) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 429, -3 }, /* (527) subquery ::= NK_LP query_expression NK_RP */ + { 429, -3 }, /* (528) subquery ::= NK_LP subquery NK_RP */ + { 432, -1 }, /* (529) search_condition ::= common_expression */ + { 455, -1 }, /* (530) sort_specification_list ::= sort_specification */ + { 455, -3 }, /* (531) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 456, -3 }, /* (532) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 457, 0 }, /* (533) ordering_specification_opt ::= */ + { 457, -1 }, /* (534) ordering_specification_opt ::= ASC */ + { 457, -1 }, /* (535) ordering_specification_opt ::= DESC */ + { 458, 0 }, /* (536) null_ordering_opt ::= */ + { 458, -2 }, /* (537) null_ordering_opt ::= NULLS FIRST */ + { 458, -2 }, /* (538) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3578,11 +3673,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,322,&yymsp[0].minor); + yy_destructor(yypParser,325,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,323,&yymsp[0].minor); + yy_destructor(yypParser,326,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3596,20 +3691,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,322,&yymsp[-2].minor); +{ yy_destructor(yypParser,325,&yymsp[-2].minor); { } - yy_destructor(yypParser,324,&yymsp[0].minor); + yy_destructor(yypParser,327,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,325,&yymsp[0].minor); +{ yy_destructor(yypParser,328,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,323,&yymsp[-1].minor); +{ yy_destructor(yypParser,326,&yymsp[-1].minor); { } - yy_destructor(yypParser,325,&yymsp[0].minor); + yy_destructor(yypParser,328,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3623,80 +3718,80 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,324,&yymsp[0].minor); + yy_destructor(yypParser,327,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy737, &yymsp[-1].minor.yy0, yymsp[0].minor.yy695); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy317, &yymsp[-1].minor.yy0, yymsp[0].minor.yy449); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy737, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy737, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy737, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy317, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy317); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy695 = 1; } +{ yymsp[1].minor.yy449 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy695 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy449 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy93, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy531, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy93, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy531, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy93 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy93 = yymsp[0].minor.yy93; } - yymsp[0].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy531 = yymsp[0].minor.yy531; } + yymsp[0].minor.yy531 = yylhsminor.yy531; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy93 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy93 = yymsp[-2].minor.yy93 | yymsp[0].minor.yy93; } - yymsp[-2].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy531 = yymsp[-2].minor.yy531 | yymsp[0].minor.yy531; } + yymsp[-2].minor.yy531 = yylhsminor.yy531; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy93 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy93 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy531 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy737 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy737 = yylhsminor.yy737; +{ yylhsminor.yy317 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy317 = yylhsminor.yy317; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy737 = yymsp[-2].minor.yy737; } - yymsp[-2].minor.yy737 = yylhsminor.yy737; +{ yylhsminor.yy317 = yymsp[-2].minor.yy317; } + yymsp[-2].minor.yy317 = yylhsminor.yy317; break; case 42: /* priv_level ::= topic_name */ case 455: /* alias_opt ::= table_alias */ yytestcase(yyruleno==455); -{ yylhsminor.yy737 = yymsp[0].minor.yy737; } - yymsp[0].minor.yy737 = yylhsminor.yy737; +{ yylhsminor.yy317 = yymsp[0].minor.yy317; } + yymsp[0].minor.yy317 = yylhsminor.yy317; break; case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy737, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy317, NULL); } break; case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0); } break; case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy185); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy335); } break; case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy737, yymsp[0].minor.yy185); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy335); } break; case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3736,8 +3831,8 @@ static YYACTIONTYPE yy_reduce( case 400: /* star_func ::= FIRST */ yytestcase(yyruleno==400); case 401: /* star_func ::= LAST */ yytestcase(yyruleno==401); case 402: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==402); -{ yylhsminor.yy737 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy737 = yylhsminor.yy737; +{ yylhsminor.yy317 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy317 = yylhsminor.yy317; break; case 54: /* force_opt ::= */ case 73: /* not_exists_opt ::= */ yytestcase(yyruleno==73); @@ -3745,13 +3840,13 @@ static YYACTIONTYPE yy_reduce( case 282: /* analyze_opt ::= */ yytestcase(yyruleno==282); case 289: /* agg_func_opt ::= */ yytestcase(yyruleno==289); case 463: /* set_quantifier_opt ::= */ yytestcase(yyruleno==463); -{ yymsp[1].minor.yy185 = false; } +{ yymsp[1].minor.yy335 = false; } break; case 55: /* force_opt ::= FORCE */ case 283: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==283); case 290: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==290); case 464: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==464); -{ yymsp[0].minor.yy185 = true; } +{ yymsp[0].minor.yy335 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3784,761 +3879,762 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy185, &yymsp[-1].minor.yy737, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy335, &yymsp[-1].minor.yy317, yymsp[0].minor.yy74); } break; case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy185, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } break; case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } break; case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy737, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy74); } break; case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } break; case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy737, yymsp[0].minor.yy196); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy856); } break; case 72: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy185 = true; } +{ yymsp[-2].minor.yy335 = true; } break; case 74: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy185 = true; } +{ yymsp[-1].minor.yy335 = true; } break; case 76: /* db_options ::= */ -{ yymsp[1].minor.yy104 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy74 = createDefaultDatabaseOptions(pCxt); } break; case 77: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 78: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 79: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 80: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 81: /* db_options ::= db_options DURATION NK_INTEGER */ case 82: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==82); -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 83: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 84: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 85: /* db_options ::= db_options KEEP integer_list */ case 86: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==86); -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_KEEP, yymsp[0].minor.yy616); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_KEEP, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 87: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 88: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 89: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 90: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 91: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 92: /* db_options ::= db_options STRICT NK_STRING */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 92: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 93: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 93: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 94: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 94: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_RETENTIONS, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 95: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_RETENTIONS, yymsp[0].minor.yy616); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 95: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 96: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 96: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 97: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 97: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 98: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 98: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; - break; - case 100: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-3].minor.yy104, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-3].minor.yy74, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 100: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 102: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-3].minor.yy104, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-3].minor.yy74, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; - break; - case 103: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 104: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 102: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 105: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 103: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 106: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 104: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 107: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy104 = setDatabaseOption(pCxt, yymsp[-2].minor.yy104, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 105: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 108: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy104 = createAlterDatabaseOptions(pCxt); yylhsminor.yy104 = setAlterDatabaseOption(pCxt, yylhsminor.yy104, &yymsp[0].minor.yy557); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 106: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ +{ yylhsminor.yy74 = setDatabaseOption(pCxt, yymsp[-2].minor.yy74, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 109: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy104 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy104, &yymsp[0].minor.yy557); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 107: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy74 = createAlterDatabaseOptions(pCxt); yylhsminor.yy74 = setAlterDatabaseOption(pCxt, yylhsminor.yy74, &yymsp[0].minor.yy767); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 110: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 108: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy74 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy74, &yymsp[0].minor.yy767); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 111: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 109: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 112: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 110: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 113: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 111: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 114: /* alter_db_option ::= KEEP integer_list */ - case 115: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==115); -{ yymsp[-1].minor.yy557.type = DB_OPTION_KEEP; yymsp[-1].minor.yy557.pList = yymsp[0].minor.yy616; } + case 112: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 116: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_PAGES; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 113: /* alter_db_option ::= KEEP integer_list */ + case 114: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==114); +{ yymsp[-1].minor.yy767.type = DB_OPTION_KEEP; yymsp[-1].minor.yy767.pList = yymsp[0].minor.yy874; } break; - case 117: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 115: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_PAGES; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 118: /* alter_db_option ::= STRICT NK_STRING */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_STRICT; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 116: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 119: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_WAL; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 117: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_WAL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 120: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 118: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 121: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy616 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy616 = yylhsminor.yy616; + case 119: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 122: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 120: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ case 312: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==312); -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-2].minor.yy616, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy616 = yylhsminor.yy616; - break; - case 123: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy616 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy616 = yylhsminor.yy616; - break; - case 124: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-2].minor.yy616, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy616 = yylhsminor.yy616; - break; - case 125: /* retention_list ::= retention */ - case 147: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==147); - case 150: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==150); - case 157: /* column_def_list ::= column_def */ yytestcase(yyruleno==157); - case 200: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==200); - case 205: /* col_name_list ::= col_name */ yytestcase(yyruleno==205); - case 254: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==254); - case 265: /* func_list ::= func */ yytestcase(yyruleno==265); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; + break; + case 121: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy874 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; + break; + case 122: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; + break; + case 123: /* retention_list ::= retention */ + case 145: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==145); + case 148: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==148); + case 155: /* column_def_list ::= column_def */ yytestcase(yyruleno==155); + case 199: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==199); + case 204: /* col_name_list ::= col_name */ yytestcase(yyruleno==204); + case 253: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==253); + case 264: /* func_list ::= func */ yytestcase(yyruleno==264); case 340: /* literal_list ::= signed_literal */ yytestcase(yyruleno==340); case 405: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==405); case 411: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==411); case 466: /* select_list ::= select_item */ yytestcase(yyruleno==466); case 477: /* partition_list ::= partition_item */ yytestcase(yyruleno==477); - case 529: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==529); -{ yylhsminor.yy616 = createNodeList(pCxt, yymsp[0].minor.yy104); } - yymsp[0].minor.yy616 = yylhsminor.yy616; - break; - case 126: /* retention_list ::= retention_list NK_COMMA retention */ - case 158: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==158); - case 201: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==201); - case 206: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==206); - case 255: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==255); - case 266: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==266); + case 530: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==530); +{ yylhsminor.yy874 = createNodeList(pCxt, yymsp[0].minor.yy74); } + yymsp[0].minor.yy874 = yylhsminor.yy874; + break; + case 124: /* retention_list ::= retention_list NK_COMMA retention */ + case 156: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==156); + case 200: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==200); + case 205: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==205); + case 254: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==254); + case 265: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==265); case 341: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==341); case 406: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==406); case 467: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==467); case 478: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==478); - case 530: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==530); -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-2].minor.yy616, yymsp[0].minor.yy104); } - yymsp[-2].minor.yy616 = yylhsminor.yy616; + case 531: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==531); +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; - case 127: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy104 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 125: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy74 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 128: /* speed_opt ::= */ + case 126: /* speed_opt ::= */ case 291: /* bufsize_opt ::= */ yytestcase(yyruleno==291); -{ yymsp[1].minor.yy196 = 0; } +{ yymsp[1].minor.yy856 = 0; } break; - case 129: /* speed_opt ::= MAX_SPEED NK_INTEGER */ + case 127: /* speed_opt ::= MAX_SPEED NK_INTEGER */ case 292: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==292); -{ yymsp[-1].minor.yy196 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy856 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 130: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 132: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==132); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy185, yymsp[-5].minor.yy104, yymsp[-3].minor.yy616, yymsp[-1].minor.yy616, yymsp[0].minor.yy104); } + case 128: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 130: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==130); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy335, yymsp[-5].minor.yy74, yymsp[-3].minor.yy874, yymsp[-1].minor.yy874, yymsp[0].minor.yy74); } break; - case 131: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy616); } + case 129: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy874); } break; - case 133: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy616); } + case 131: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy874); } break; - case 134: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy185, yymsp[0].minor.yy104); } + case 132: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } break; - case 135: /* cmd ::= ALTER TABLE alter_table_clause */ + case 133: /* cmd ::= ALTER TABLE alter_table_clause */ case 314: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==314); -{ pCxt->pRootNode = yymsp[0].minor.yy104; } +{ pCxt->pRootNode = yymsp[0].minor.yy74; } break; - case 136: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy104); } + case 134: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy74); } break; - case 137: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy104 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 135: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy74 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 138: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy104 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy737, yymsp[0].minor.yy640); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 136: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 139: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy104 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy104, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy737); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + case 137: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy74 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy74, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy317); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 140: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy104 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy737, yymsp[0].minor.yy640); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 138: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 141: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy104 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy737, &yymsp[0].minor.yy737); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 139: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy74 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 142: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy104 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy737, yymsp[0].minor.yy640); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 140: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 143: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy104 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy104, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy737); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + case 141: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy74 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy74, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy317); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 144: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy104 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy737, yymsp[0].minor.yy640); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 142: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy74 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 145: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy104 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy104, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy737, &yymsp[0].minor.yy737); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 143: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy74 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy74, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 146: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy104 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy104, &yymsp[-2].minor.yy737, yymsp[0].minor.yy104); } - yymsp[-5].minor.yy104 = yylhsminor.yy104; + case 144: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy74 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy74, &yymsp[-2].minor.yy317, yymsp[0].minor.yy74); } + yymsp[-5].minor.yy74 = yylhsminor.yy74; break; - case 148: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 151: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==151); + case 146: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 149: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==149); case 412: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==412); -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-1].minor.yy616, yymsp[0].minor.yy104); } - yymsp[-1].minor.yy616 = yylhsminor.yy616; +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-1].minor.yy874, yymsp[0].minor.yy74); } + yymsp[-1].minor.yy874 = yylhsminor.yy874; break; - case 149: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy104 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy185, yymsp[-8].minor.yy104, yymsp[-6].minor.yy104, yymsp[-5].minor.yy616, yymsp[-2].minor.yy616, yymsp[0].minor.yy104); } - yymsp[-9].minor.yy104 = yylhsminor.yy104; + case 147: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy74 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy335, yymsp[-8].minor.yy74, yymsp[-6].minor.yy74, yymsp[-5].minor.yy874, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } + yymsp[-9].minor.yy74 = yylhsminor.yy74; break; - case 152: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy104 = createDropTableClause(pCxt, yymsp[-1].minor.yy185, yymsp[0].minor.yy104); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 150: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy74 = createDropTableClause(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 153: /* specific_cols_opt ::= */ - case 184: /* tags_def_opt ::= */ yytestcase(yyruleno==184); - case 253: /* tag_list_opt ::= */ yytestcase(yyruleno==253); + case 151: /* specific_cols_opt ::= */ + case 182: /* tags_def_opt ::= */ yytestcase(yyruleno==182); + case 252: /* tag_list_opt ::= */ yytestcase(yyruleno==252); case 475: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==475); - case 497: /* group_by_clause_opt ::= */ yytestcase(yyruleno==497); - case 516: /* order_by_clause_opt ::= */ yytestcase(yyruleno==516); -{ yymsp[1].minor.yy616 = NULL; } + case 498: /* group_by_clause_opt ::= */ yytestcase(yyruleno==498); + case 517: /* order_by_clause_opt ::= */ yytestcase(yyruleno==517); +{ yymsp[1].minor.yy874 = NULL; } break; - case 154: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy616 = yymsp[-1].minor.yy616; } + case 152: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy874 = yymsp[-1].minor.yy874; } break; - case 155: /* full_table_name ::= table_name */ -{ yylhsminor.yy104 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy737, NULL); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 153: /* full_table_name ::= table_name */ +{ yylhsminor.yy74 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy317, NULL); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 156: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy104 = createRealTableNode(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737, NULL); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 154: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy74 = createRealTableNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317, NULL); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 159: /* column_def ::= column_name type_name */ -{ yylhsminor.yy104 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy737, yymsp[0].minor.yy640, NULL); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 157: /* column_def ::= column_name type_name */ +{ yylhsminor.yy74 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy317, yymsp[0].minor.yy898, NULL); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 160: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy104 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy737, yymsp[-2].minor.yy640, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + case 158: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy74 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-2].minor.yy898, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 161: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 159: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 162: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 160: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 163: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 161: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 164: /* type_name ::= INT */ - case 165: /* type_name ::= INTEGER */ yytestcase(yyruleno==165); -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_INT); } + case 162: /* type_name ::= INT */ + case 163: /* type_name ::= INTEGER */ yytestcase(yyruleno==163); +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 166: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 164: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 167: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 165: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 168: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 166: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 169: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy640 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 167: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 170: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 168: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 171: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy640 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 169: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 172: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy640 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 170: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 173: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy640 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 171: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 174: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy640 = createDataType(TSDB_DATA_TYPE_UINT); } + case 172: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 175: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy640 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 173: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy898 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 176: /* type_name ::= JSON */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_JSON); } + case 174: /* type_name ::= JSON */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 177: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy640 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 175: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 178: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 176: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 179: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 177: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 180: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy640 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 178: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy898 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 181: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy640 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 179: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 182: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy640 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 180: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 183: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy640 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy898 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 185: /* tags_def_opt ::= tags_def */ + case 183: /* tags_def_opt ::= tags_def */ case 404: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==404); -{ yylhsminor.yy616 = yymsp[0].minor.yy616; } - yymsp[0].minor.yy616 = yylhsminor.yy616; +{ yylhsminor.yy874 = yymsp[0].minor.yy874; } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 186: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy616 = yymsp[-1].minor.yy616; } + case 184: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy874 = yymsp[-1].minor.yy874; } break; - case 187: /* table_options ::= */ -{ yymsp[1].minor.yy104 = createDefaultTableOptions(pCxt); } + case 185: /* table_options ::= */ +{ yymsp[1].minor.yy74 = createDefaultTableOptions(pCxt); } break; - case 188: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-2].minor.yy104, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 186: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 189: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-2].minor.yy104, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy616); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 187: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 190: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-2].minor.yy104, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy616); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 188: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 191: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-4].minor.yy104, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy616); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 189: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-4].minor.yy74, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 192: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-2].minor.yy104, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 190: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 193: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-4].minor.yy104, TABLE_OPTION_SMA, yymsp[-1].minor.yy616); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + case 191: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-4].minor.yy74, TABLE_OPTION_SMA, yymsp[-1].minor.yy874); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; - case 194: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy104 = createAlterTableOptions(pCxt); yylhsminor.yy104 = setTableOption(pCxt, yylhsminor.yy104, yymsp[0].minor.yy557.type, &yymsp[0].minor.yy557.val); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 192: /* table_options ::= table_options DELETE_MARK duration_list */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-2].minor.yy74, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy874); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 195: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy104 = setTableOption(pCxt, yymsp[-1].minor.yy104, yymsp[0].minor.yy557.type, &yymsp[0].minor.yy557.val); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 193: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy74 = createAlterTableOptions(pCxt); yylhsminor.yy74 = setTableOption(pCxt, yylhsminor.yy74, yymsp[0].minor.yy767.type, &yymsp[0].minor.yy767.val); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 196: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy557.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 194: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy74 = setTableOption(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy767.type, &yymsp[0].minor.yy767.val); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 197: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy557.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy557.val = yymsp[0].minor.yy0; } + case 195: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy767.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } break; - case 198: /* duration_list ::= duration_literal */ + case 196: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy767.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy767.val = yymsp[0].minor.yy0; } + break; + case 197: /* duration_list ::= duration_literal */ case 369: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==369); -{ yylhsminor.yy616 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104)); } - yymsp[0].minor.yy616 = yylhsminor.yy616; +{ yylhsminor.yy874 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 199: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 198: /* duration_list ::= duration_list NK_COMMA duration_literal */ case 370: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==370); -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-2].minor.yy616, releaseRawExprNode(pCxt, yymsp[0].minor.yy104)); } - yymsp[-2].minor.yy616 = yylhsminor.yy616; +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; - case 202: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy104 = createFunctionNode(pCxt, &yymsp[0].minor.yy737, NULL); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 201: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[0].minor.yy317, NULL); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 203: /* rollup_func_name ::= FIRST */ - case 204: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==204); - case 257: /* tag_item ::= QTAGS */ yytestcase(yyruleno==257); -{ yylhsminor.yy104 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 202: /* rollup_func_name ::= FIRST */ + case 203: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==203); + case 256: /* tag_item ::= QTAGS */ yytestcase(yyruleno==256); +{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 207: /* col_name ::= column_name */ - case 258: /* tag_item ::= column_name */ yytestcase(yyruleno==258); -{ yylhsminor.yy104 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy737); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 206: /* col_name ::= column_name */ + case 257: /* tag_item ::= column_name */ yytestcase(yyruleno==257); +{ yylhsminor.yy74 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy317); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 208: /* cmd ::= SHOW DNODES */ + case 207: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 209: /* cmd ::= SHOW USERS */ + case 208: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 210: /* cmd ::= SHOW USER PRIVILEGES */ + case 209: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } break; - case 211: /* cmd ::= SHOW DATABASES */ + case 210: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 212: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy104, yymsp[0].minor.yy104, OP_TYPE_LIKE); } + case 211: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, OP_TYPE_LIKE); } break; - case 213: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy104, yymsp[0].minor.yy104, OP_TYPE_LIKE); } + case 212: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, OP_TYPE_LIKE); } break; - case 214: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy104, NULL, OP_TYPE_LIKE); } + case 213: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy74, NULL, OP_TYPE_LIKE); } break; - case 215: /* cmd ::= SHOW MNODES */ + case 214: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 216: /* cmd ::= SHOW QNODES */ + case 215: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 217: /* cmd ::= SHOW FUNCTIONS */ + case 216: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 218: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy104, yymsp[-1].minor.yy104, OP_TYPE_EQUAL); } + case 217: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy74, yymsp[-1].minor.yy74, OP_TYPE_EQUAL); } break; - case 219: /* cmd ::= SHOW STREAMS */ + case 218: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 220: /* cmd ::= SHOW ACCOUNTS */ + case 219: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 221: /* cmd ::= SHOW APPS */ + case 220: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 222: /* cmd ::= SHOW CONNECTIONS */ + case 221: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 223: /* cmd ::= SHOW LICENCES */ - case 224: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==224); + case 222: /* cmd ::= SHOW LICENCES */ + case 223: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==223); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 225: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy737); } + case 224: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy317); } break; - case 226: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy104); } + case 225: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy74); } break; - case 227: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy104); } + case 226: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy74); } break; - case 228: /* cmd ::= SHOW QUERIES */ + case 227: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 229: /* cmd ::= SHOW SCORES */ + case 228: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 230: /* cmd ::= SHOW TOPICS */ + case 229: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 231: /* cmd ::= SHOW VARIABLES */ - case 232: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==232); + case 230: /* cmd ::= SHOW VARIABLES */ + case 231: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==231); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 233: /* cmd ::= SHOW LOCAL VARIABLES */ + case 232: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 234: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy104); } + case 233: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy74); } break; - case 235: /* cmd ::= SHOW BNODES */ + case 234: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 236: /* cmd ::= SHOW SNODES */ + case 235: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 237: /* cmd ::= SHOW CLUSTER */ + case 236: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 238: /* cmd ::= SHOW TRANSACTIONS */ + case 237: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 239: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy104); } + case 238: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy74); } break; - case 240: /* cmd ::= SHOW CONSUMERS */ + case 239: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 241: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 240: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 242: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy104, yymsp[-1].minor.yy104, OP_TYPE_EQUAL); } + case 241: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy74, yymsp[-1].minor.yy74, OP_TYPE_EQUAL); } break; - case 243: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy104, yymsp[0].minor.yy104, yymsp[-3].minor.yy616); } + case 242: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74, yymsp[-3].minor.yy874); } break; - case 244: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 243: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 245: /* cmd ::= SHOW VNODES NK_STRING */ + case 244: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 246: /* db_name_cond_opt ::= */ - case 251: /* from_db_opt ::= */ yytestcase(yyruleno==251); -{ yymsp[1].minor.yy104 = createDefaultDatabaseCondValue(pCxt); } + case 245: /* db_name_cond_opt ::= */ + case 250: /* from_db_opt ::= */ yytestcase(yyruleno==250); +{ yymsp[1].minor.yy74 = createDefaultDatabaseCondValue(pCxt); } break; - case 247: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy104 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy737); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 246: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy74 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy317); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 248: /* like_pattern_opt ::= */ + case 247: /* like_pattern_opt ::= */ case 302: /* subtable_opt ::= */ yytestcase(yyruleno==302); case 414: /* case_when_else_opt ::= */ yytestcase(yyruleno==414); case 444: /* from_clause_opt ::= */ yytestcase(yyruleno==444); case 473: /* where_clause_opt ::= */ yytestcase(yyruleno==473); case 482: /* twindow_clause_opt ::= */ yytestcase(yyruleno==482); - case 487: /* sliding_opt ::= */ yytestcase(yyruleno==487); - case 489: /* fill_opt ::= */ yytestcase(yyruleno==489); - case 501: /* having_clause_opt ::= */ yytestcase(yyruleno==501); - case 503: /* range_opt ::= */ yytestcase(yyruleno==503); - case 505: /* every_opt ::= */ yytestcase(yyruleno==505); - case 518: /* slimit_clause_opt ::= */ yytestcase(yyruleno==518); - case 522: /* limit_clause_opt ::= */ yytestcase(yyruleno==522); -{ yymsp[1].minor.yy104 = NULL; } + case 488: /* sliding_opt ::= */ yytestcase(yyruleno==488); + case 490: /* fill_opt ::= */ yytestcase(yyruleno==490); + case 502: /* having_clause_opt ::= */ yytestcase(yyruleno==502); + case 504: /* range_opt ::= */ yytestcase(yyruleno==504); + case 506: /* every_opt ::= */ yytestcase(yyruleno==506); + case 519: /* slimit_clause_opt ::= */ yytestcase(yyruleno==519); + case 523: /* limit_clause_opt ::= */ yytestcase(yyruleno==523); +{ yymsp[1].minor.yy74 = NULL; } break; - case 249: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 248: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 250: /* table_name_cond ::= table_name */ -{ yylhsminor.yy104 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy737); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 249: /* table_name_cond ::= table_name */ +{ yylhsminor.yy74 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy317); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 252: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy104 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy737); } + case 251: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy74 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy317); } break; - case 256: /* tag_item ::= TBNAME */ -{ yylhsminor.yy104 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 255: /* tag_item ::= TBNAME */ +{ yylhsminor.yy74 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; - case 259: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy104 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy737), &yymsp[0].minor.yy737); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + case 258: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy74 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy317), &yymsp[0].minor.yy317); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; - case 260: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy104 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy737), &yymsp[0].minor.yy737); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 259: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy74 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy317), &yymsp[0].minor.yy317); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 261: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy185, yymsp[-3].minor.yy104, yymsp[-1].minor.yy104, NULL, yymsp[0].minor.yy104); } + case 260: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy335, yymsp[-3].minor.yy74, yymsp[-1].minor.yy74, NULL, yymsp[0].minor.yy74); } break; - case 262: /* cmd ::= DROP INDEX exists_opt full_table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy185, yymsp[0].minor.yy104); } + case 261: /* cmd ::= DROP INDEX exists_opt full_table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy335, yymsp[0].minor.yy74); } break; - case 263: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy104 = createIndexOption(pCxt, yymsp[-7].minor.yy616, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), NULL, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } + case 262: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy74 = createIndexOption(pCxt, yymsp[-7].minor.yy874, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } break; - case 264: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy104 = createIndexOption(pCxt, yymsp[-9].minor.yy616, releaseRawExprNode(pCxt, yymsp[-5].minor.yy104), releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } + case 263: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy74 = createIndexOption(pCxt, yymsp[-9].minor.yy874, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } break; - case 267: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy104 = createFunctionNode(pCxt, &yymsp[-3].minor.yy737, yymsp[-1].minor.yy616); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + case 266: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy74 = createFunctionNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-1].minor.yy874); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 268: /* sma_stream_opt ::= */ + case 267: /* sma_stream_opt ::= */ case 295: /* stream_options ::= */ yytestcase(yyruleno==295); -{ yymsp[1].minor.yy104 = createStreamOptions(pCxt); } +{ yymsp[1].minor.yy74 = createStreamOptions(pCxt); } break; - case 269: /* sma_stream_opt ::= stream_options WATERMARK duration_literal */ + case 268: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ case 299: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==299); -{ ((SStreamOptions*)yymsp[-2].minor.yy104)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); yylhsminor.yy104 = yymsp[-2].minor.yy104; } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; + break; + case 269: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 270: /* sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy104)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); yylhsminor.yy104 = yymsp[-2].minor.yy104; } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 270: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 271: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy185, &yymsp[-2].minor.yy737, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy335, &yymsp[-2].minor.yy317, yymsp[0].minor.yy74); } break; case 272: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy185, &yymsp[-3].minor.yy737, &yymsp[0].minor.yy737, false); } +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy335, &yymsp[-3].minor.yy317, &yymsp[0].minor.yy317, false); } break; case 273: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy185, &yymsp[-5].minor.yy737, &yymsp[0].minor.yy737, true); } +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy335, &yymsp[-5].minor.yy317, &yymsp[0].minor.yy317, true); } break; case 274: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy185, &yymsp[-3].minor.yy737, yymsp[0].minor.yy104, false); } +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy335, &yymsp[-3].minor.yy317, yymsp[0].minor.yy74, false); } break; case 275: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy185, &yymsp[-5].minor.yy737, yymsp[0].minor.yy104, true); } +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy335, &yymsp[-5].minor.yy317, yymsp[0].minor.yy74, true); } break; case 276: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy185, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } break; case 277: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy185, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy335, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317); } break; case 278: /* cmd ::= DESC full_table_name */ case 279: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==279); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy74); } break; case 280: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; case 281: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy185, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy335, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } break; case 284: /* explain_options ::= */ -{ yymsp[1].minor.yy104 = createDefaultExplainOptions(pCxt); } +{ yymsp[1].minor.yy74 = createDefaultExplainOptions(pCxt); } break; case 285: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy104 = setExplainVerbose(pCxt, yymsp[-2].minor.yy104, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setExplainVerbose(pCxt, yymsp[-2].minor.yy74, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 286: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy104 = setExplainRatio(pCxt, yymsp[-2].minor.yy104, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setExplainRatio(pCxt, yymsp[-2].minor.yy74, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 287: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy185, yymsp[-8].minor.yy185, &yymsp[-5].minor.yy737, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy640, yymsp[0].minor.yy196); } +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy335, yymsp[-8].minor.yy335, &yymsp[-5].minor.yy317, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy898, yymsp[0].minor.yy856); } break; case 288: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy185, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } break; case 293: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy185, &yymsp[-7].minor.yy737, yymsp[-4].minor.yy104, yymsp[-6].minor.yy104, yymsp[-3].minor.yy616, yymsp[-2].minor.yy104, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy335, &yymsp[-7].minor.yy317, yymsp[-4].minor.yy74, yymsp[-6].minor.yy74, yymsp[-3].minor.yy874, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } break; case 294: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy185, &yymsp[0].minor.yy737); } +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy335, &yymsp[0].minor.yy317); } break; case 296: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy104)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy104 = yymsp[-2].minor.yy104; } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 297: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy104)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy104 = yymsp[-2].minor.yy104; } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 298: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy104)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy104)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); yylhsminor.yy104 = yymsp[-3].minor.yy104; } - yymsp[-3].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-3].minor.yy74)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy74)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); yylhsminor.yy74 = yymsp[-3].minor.yy74; } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 300: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy104)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy104 = yymsp[-3].minor.yy104; } - yymsp[-3].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-3].minor.yy74)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy74 = yymsp[-3].minor.yy74; } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 301: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-2].minor.yy104)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy104 = yymsp[-2].minor.yy104; } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ ((SStreamOptions*)yymsp[-2].minor.yy74)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy74 = yymsp[-2].minor.yy74; } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 303: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 488: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==488); - case 506: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==506); -{ yymsp[-3].minor.yy104 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy104); } + case 489: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==489); + case 507: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==507); +{ yymsp[-3].minor.yy74 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy74); } break; case 304: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } @@ -4556,42 +4652,42 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 309: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy616); } +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy874); } break; case 310: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; case 311: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy616 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +{ yymsp[-1].minor.yy874 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; case 313: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } break; case 315: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy104, yymsp[-2].minor.yy616, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy74, yymsp[-2].minor.yy874, yymsp[0].minor.yy74); } break; case 316: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy104, NULL, yymsp[0].minor.yy104); } +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy74, NULL, yymsp[0].minor.yy74); } break; case 317: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 318: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 319: /* literal ::= NK_STRING */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 320: /* literal ::= NK_BOOL */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 321: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 322: /* literal ::= duration_literal */ case 332: /* signed_literal ::= signed */ yytestcase(yyruleno==332); @@ -4610,175 +4706,175 @@ static YYACTIONTYPE yy_reduce( case 448: /* table_reference ::= table_primary */ yytestcase(yyruleno==448); case 449: /* table_reference ::= joined_table */ yytestcase(yyruleno==449); case 453: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==453); - case 508: /* query_simple ::= query_specification */ yytestcase(yyruleno==508); - case 509: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==509); - case 512: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==512); - case 514: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==514); -{ yylhsminor.yy104 = yymsp[0].minor.yy104; } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 509: /* query_simple ::= query_specification */ yytestcase(yyruleno==509); + case 510: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==510); + case 513: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==513); + case 515: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==515); +{ yylhsminor.yy74 = yymsp[0].minor.yy74; } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 323: /* literal ::= NULL */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 324: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 325: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 326: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 327: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } break; case 328: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 329: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 330: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; case 331: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 333: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 334: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 335: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; case 336: /* signed_literal ::= duration_literal */ case 338: /* signed_literal ::= literal_func */ yytestcase(yyruleno==338); case 407: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==407); case 469: /* select_item ::= common_expression */ yytestcase(yyruleno==469); case 479: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==479); - case 513: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==513); - case 515: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==515); - case 528: /* search_condition ::= common_expression */ yytestcase(yyruleno==528); -{ yylhsminor.yy104 = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); } - yymsp[0].minor.yy104 = yylhsminor.yy104; + case 514: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==514); + case 516: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==516); + case 529: /* search_condition ::= common_expression */ yytestcase(yyruleno==529); +{ yylhsminor.yy74 = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 337: /* signed_literal ::= NULL */ -{ yylhsminor.yy104 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 339: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy104 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 358: /* expression ::= NK_LP expression NK_RP */ case 441: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==441); - case 527: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==527); -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 528: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==528); +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 359: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy104)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 360: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy74), NULL)); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 361: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 362: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 363: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 364: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 365: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 366: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 367: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 368: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 371: /* column_reference ::= column_name */ -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy737, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy737)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy317, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy317)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 372: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737, createColumnNode(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy737)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317, createColumnNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy317)); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 373: /* pseudo_column ::= ROWTS */ case 374: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==374); @@ -4791,327 +4887,330 @@ static YYACTIONTYPE yy_reduce( case 382: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==382); case 383: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==383); case 389: /* literal_func ::= NOW */ yytestcase(yyruleno==389); -{ yylhsminor.yy104 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 375: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy737)))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy317)))); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 384: /* function_expression ::= function_name NK_LP expression_list NK_RP */ case 385: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==385); -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy737, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy737, yymsp[-1].minor.yy616)); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy317, yymsp[-1].minor.yy874)); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 386: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), yymsp[-1].minor.yy640)); } - yymsp[-5].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy898)); } + yymsp[-5].minor.yy74 = yylhsminor.yy74; break; case 388: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy737, NULL)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy317, NULL)); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 403: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy616 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy616 = yylhsminor.yy616; +{ yylhsminor.yy874 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; case 408: /* star_func_para ::= table_name NK_DOT NK_STAR */ case 472: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==472); -{ yylhsminor.yy104 = createColumnNode(pCxt, &yymsp[-2].minor.yy737, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createColumnNode(pCxt, &yymsp[-2].minor.yy317, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 409: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy616, yymsp[-1].minor.yy104)); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy874, yymsp[-1].minor.yy74)); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 410: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), yymsp[-2].minor.yy616, yymsp[-1].minor.yy104)); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-2].minor.yy874, yymsp[-1].minor.yy74)); } + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; case 413: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy104 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104)); } +{ yymsp[-3].minor.yy74 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74)); } break; case 415: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy104 = releaseRawExprNode(pCxt, yymsp[0].minor.yy104); } +{ yymsp[-1].minor.yy74 = releaseRawExprNode(pCxt, yymsp[0].minor.yy74); } break; case 416: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ case 421: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==421); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy668, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy20, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 417: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy104), releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy74), releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-4].minor.yy104 = yylhsminor.yy104; + yymsp[-4].minor.yy74 = yylhsminor.yy74; break; case 418: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy104), releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-5].minor.yy104 = yylhsminor.yy104; + yymsp[-5].minor.yy74 = yylhsminor.yy74; break; case 419: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), NULL)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 420: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL)); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 422: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy668 = OP_TYPE_LOWER_THAN; } +{ yymsp[0].minor.yy20 = OP_TYPE_LOWER_THAN; } break; case 423: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy668 = OP_TYPE_GREATER_THAN; } +{ yymsp[0].minor.yy20 = OP_TYPE_GREATER_THAN; } break; case 424: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy668 = OP_TYPE_LOWER_EQUAL; } +{ yymsp[0].minor.yy20 = OP_TYPE_LOWER_EQUAL; } break; case 425: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy668 = OP_TYPE_GREATER_EQUAL; } +{ yymsp[0].minor.yy20 = OP_TYPE_GREATER_EQUAL; } break; case 426: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy668 = OP_TYPE_NOT_EQUAL; } +{ yymsp[0].minor.yy20 = OP_TYPE_NOT_EQUAL; } break; case 427: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy668 = OP_TYPE_EQUAL; } +{ yymsp[0].minor.yy20 = OP_TYPE_EQUAL; } break; case 428: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy668 = OP_TYPE_LIKE; } +{ yymsp[0].minor.yy20 = OP_TYPE_LIKE; } break; case 429: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy668 = OP_TYPE_NOT_LIKE; } +{ yymsp[-1].minor.yy20 = OP_TYPE_NOT_LIKE; } break; case 430: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy668 = OP_TYPE_MATCH; } +{ yymsp[0].minor.yy20 = OP_TYPE_MATCH; } break; case 431: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy668 = OP_TYPE_NMATCH; } +{ yymsp[0].minor.yy20 = OP_TYPE_NMATCH; } break; case 432: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy668 = OP_TYPE_JSON_CONTAINS; } +{ yymsp[0].minor.yy20 = OP_TYPE_JSON_CONTAINS; } break; case 433: /* in_op ::= IN */ -{ yymsp[0].minor.yy668 = OP_TYPE_IN; } +{ yymsp[0].minor.yy20 = OP_TYPE_IN; } break; case 434: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy668 = OP_TYPE_NOT_IN; } +{ yymsp[-1].minor.yy20 = OP_TYPE_NOT_IN; } break; case 435: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy616)); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 437: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy104), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy74), NULL)); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 438: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 439: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy104); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy104); - yylhsminor.yy104 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy74); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy74); + yylhsminor.yy74 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 445: /* from_clause_opt ::= FROM table_reference_list */ case 474: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==474); - case 502: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==502); -{ yymsp[-1].minor.yy104 = yymsp[0].minor.yy104; } + case 503: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==503); +{ yymsp[-1].minor.yy74 = yymsp[0].minor.yy74; } break; case 447: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy104 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy104, yymsp[0].minor.yy104, NULL); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy74, yymsp[0].minor.yy74, NULL); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 450: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy104 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy737, &yymsp[0].minor.yy737); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 451: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy104 = createRealTableNode(pCxt, &yymsp[-3].minor.yy737, &yymsp[-1].minor.yy737, &yymsp[0].minor.yy737); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createRealTableNode(pCxt, &yymsp[-3].minor.yy317, &yymsp[-1].minor.yy317, &yymsp[0].minor.yy317); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; case 452: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy104 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104), &yymsp[0].minor.yy737); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74), &yymsp[0].minor.yy317); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 454: /* alias_opt ::= */ -{ yymsp[1].minor.yy737 = nil_token; } +{ yymsp[1].minor.yy317 = nil_token; } break; case 456: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy737 = yymsp[0].minor.yy737; } +{ yymsp[-1].minor.yy317 = yymsp[0].minor.yy317; } break; case 457: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ case 458: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==458); -{ yymsp[-2].minor.yy104 = yymsp[-1].minor.yy104; } +{ yymsp[-2].minor.yy74 = yymsp[-1].minor.yy74; } break; case 459: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy104 = createJoinTableNode(pCxt, yymsp[-4].minor.yy228, yymsp[-5].minor.yy104, yymsp[-2].minor.yy104, yymsp[0].minor.yy104); } - yymsp[-5].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createJoinTableNode(pCxt, yymsp[-4].minor.yy630, yymsp[-5].minor.yy74, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } + yymsp[-5].minor.yy74 = yylhsminor.yy74; break; case 460: /* join_type ::= */ -{ yymsp[1].minor.yy228 = JOIN_TYPE_INNER; } +{ yymsp[1].minor.yy630 = JOIN_TYPE_INNER; } break; case 461: /* join_type ::= INNER */ -{ yymsp[0].minor.yy228 = JOIN_TYPE_INNER; } +{ yymsp[0].minor.yy630 = JOIN_TYPE_INNER; } break; case 462: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy104 = createSelectStmt(pCxt, yymsp[-10].minor.yy185, yymsp[-9].minor.yy616, yymsp[-8].minor.yy104); - yymsp[-11].minor.yy104 = addWhereClause(pCxt, yymsp[-11].minor.yy104, yymsp[-7].minor.yy104); - yymsp[-11].minor.yy104 = addPartitionByClause(pCxt, yymsp[-11].minor.yy104, yymsp[-6].minor.yy616); - yymsp[-11].minor.yy104 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy104, yymsp[-2].minor.yy104); - yymsp[-11].minor.yy104 = addGroupByClause(pCxt, yymsp[-11].minor.yy104, yymsp[-1].minor.yy616); - yymsp[-11].minor.yy104 = addHavingClause(pCxt, yymsp[-11].minor.yy104, yymsp[0].minor.yy104); - yymsp[-11].minor.yy104 = addRangeClause(pCxt, yymsp[-11].minor.yy104, yymsp[-5].minor.yy104); - yymsp[-11].minor.yy104 = addEveryClause(pCxt, yymsp[-11].minor.yy104, yymsp[-4].minor.yy104); - yymsp[-11].minor.yy104 = addFillClause(pCxt, yymsp[-11].minor.yy104, yymsp[-3].minor.yy104); + yymsp[-11].minor.yy74 = createSelectStmt(pCxt, yymsp[-10].minor.yy335, yymsp[-9].minor.yy874, yymsp[-8].minor.yy74); + yymsp[-11].minor.yy74 = addWhereClause(pCxt, yymsp[-11].minor.yy74, yymsp[-7].minor.yy74); + yymsp[-11].minor.yy74 = addPartitionByClause(pCxt, yymsp[-11].minor.yy74, yymsp[-6].minor.yy874); + yymsp[-11].minor.yy74 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy74, yymsp[-2].minor.yy74); + yymsp[-11].minor.yy74 = addGroupByClause(pCxt, yymsp[-11].minor.yy74, yymsp[-1].minor.yy874); + yymsp[-11].minor.yy74 = addHavingClause(pCxt, yymsp[-11].minor.yy74, yymsp[0].minor.yy74); + yymsp[-11].minor.yy74 = addRangeClause(pCxt, yymsp[-11].minor.yy74, yymsp[-5].minor.yy74); + yymsp[-11].minor.yy74 = addEveryClause(pCxt, yymsp[-11].minor.yy74, yymsp[-4].minor.yy74); + yymsp[-11].minor.yy74 = addFillClause(pCxt, yymsp[-11].minor.yy74, yymsp[-3].minor.yy74); } break; case 465: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy185 = false; } +{ yymsp[0].minor.yy335 = false; } break; case 468: /* select_item ::= NK_STAR */ -{ yylhsminor.yy104 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy74 = yylhsminor.yy74; break; case 470: /* select_item ::= common_expression column_alias */ case 480: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==480); -{ yylhsminor.yy104 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104), &yymsp[0].minor.yy737); } - yymsp[-1].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74), &yymsp[0].minor.yy317); } + yymsp[-1].minor.yy74 = yylhsminor.yy74; break; case 471: /* select_item ::= common_expression AS column_alias */ case 481: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==481); -{ yylhsminor.yy104 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), &yymsp[0].minor.yy737); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; +{ yylhsminor.yy74 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), &yymsp[0].minor.yy317); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; case 476: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 498: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==498); - case 517: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==517); -{ yymsp[-2].minor.yy616 = yymsp[0].minor.yy616; } + case 499: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==499); + case 518: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==518); +{ yymsp[-2].minor.yy874 = yymsp[0].minor.yy874; } break; case 483: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy104 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } +{ yymsp[-5].minor.yy74 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } break; case 484: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy104 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } +{ yymsp[-3].minor.yy74 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } break; case 485: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy104 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), NULL, yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } +{ yymsp[-5].minor.yy74 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), NULL, yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } break; case 486: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy104 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy104), releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), yymsp[-1].minor.yy104, yymsp[0].minor.yy104); } +{ yymsp[-7].minor.yy74 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy74), releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), yymsp[-1].minor.yy74, yymsp[0].minor.yy74); } + break; + case 487: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy74 = createEventWindowNode(pCxt, yymsp[-3].minor.yy74, yymsp[0].minor.yy74); } break; - case 490: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy104 = createFillNode(pCxt, yymsp[-1].minor.yy246, NULL); } + case 491: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy74 = createFillNode(pCxt, yymsp[-1].minor.yy828, NULL); } break; - case 491: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy104 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy616)); } + case 492: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy74 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy874)); } break; - case 492: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy246 = FILL_MODE_NONE; } + case 493: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy828 = FILL_MODE_NONE; } break; - case 493: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy246 = FILL_MODE_PREV; } + case 494: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy828 = FILL_MODE_PREV; } break; - case 494: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy246 = FILL_MODE_NULL; } + case 495: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy828 = FILL_MODE_NULL; } break; - case 495: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy246 = FILL_MODE_LINEAR; } + case 496: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy828 = FILL_MODE_LINEAR; } break; - case 496: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy246 = FILL_MODE_NEXT; } + case 497: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy828 = FILL_MODE_NEXT; } break; - case 499: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy616 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); } - yymsp[0].minor.yy616 = yylhsminor.yy616; + case 500: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy874 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } + yymsp[0].minor.yy874 = yylhsminor.yy874; break; - case 500: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy616 = addNodeToList(pCxt, yymsp[-2].minor.yy616, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy104))); } - yymsp[-2].minor.yy616 = yylhsminor.yy616; + case 501: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy874 = addNodeToList(pCxt, yymsp[-2].minor.yy874, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy74))); } + yymsp[-2].minor.yy874 = yylhsminor.yy874; break; - case 504: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy104 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy104), releaseRawExprNode(pCxt, yymsp[-1].minor.yy104)); } + case 505: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy74 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy74), releaseRawExprNode(pCxt, yymsp[-1].minor.yy74)); } break; - case 507: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 508: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy104 = addOrderByClause(pCxt, yymsp[-3].minor.yy104, yymsp[-2].minor.yy616); - yylhsminor.yy104 = addSlimitClause(pCxt, yylhsminor.yy104, yymsp[-1].minor.yy104); - yylhsminor.yy104 = addLimitClause(pCxt, yylhsminor.yy104, yymsp[0].minor.yy104); + yylhsminor.yy74 = addOrderByClause(pCxt, yymsp[-3].minor.yy74, yymsp[-2].minor.yy874); + yylhsminor.yy74 = addSlimitClause(pCxt, yylhsminor.yy74, yymsp[-1].minor.yy74); + yylhsminor.yy74 = addLimitClause(pCxt, yylhsminor.yy74, yymsp[0].minor.yy74); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 510: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy104 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy104, yymsp[0].minor.yy104); } - yymsp[-3].minor.yy104 = yylhsminor.yy104; + case 511: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy74 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy74, yymsp[0].minor.yy74); } + yymsp[-3].minor.yy74 = yylhsminor.yy74; break; - case 511: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy104 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy104, yymsp[0].minor.yy104); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 512: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy74 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy74, yymsp[0].minor.yy74); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 519: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 523: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==523); -{ yymsp[-1].minor.yy104 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 520: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 524: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==524); +{ yymsp[-1].minor.yy74 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 520: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 524: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==524); -{ yymsp[-3].minor.yy104 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 521: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 525: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==525); +{ yymsp[-3].minor.yy74 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 521: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 525: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==525); -{ yymsp[-3].minor.yy104 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 522: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 526: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==526); +{ yymsp[-3].minor.yy74 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 526: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy104 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy104); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 527: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy74 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy74); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 531: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy104 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy104), yymsp[-1].minor.yy50, yymsp[0].minor.yy793); } - yymsp[-2].minor.yy104 = yylhsminor.yy104; + case 532: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy74 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy74), yymsp[-1].minor.yy326, yymsp[0].minor.yy109); } + yymsp[-2].minor.yy74 = yylhsminor.yy74; break; - case 532: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy50 = ORDER_ASC; } + case 533: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy326 = ORDER_ASC; } break; - case 533: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy50 = ORDER_ASC; } + case 534: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy326 = ORDER_ASC; } break; - case 534: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy50 = ORDER_DESC; } + case 535: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy326 = ORDER_DESC; } break; - case 535: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy793 = NULL_ORDER_DEFAULT; } + case 536: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy109 = NULL_ORDER_DEFAULT; } break; - case 536: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy793 = NULL_ORDER_FIRST; } + case 537: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy109 = NULL_ORDER_FIRST; } break; - case 537: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy793 = NULL_ORDER_LAST; } + case 538: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy109 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index de7487434cfc383ca5b61cbcd8d6a6e6c79c6785..ae702ec02f1570ab9d92976abdb18059cd727175 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -65,9 +65,10 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 3) .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 2) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) @@ -140,7 +141,7 @@ void generatePerformanceSchema(MockCatalogService* mcs) { void generateTestTables(MockCatalogService* mcs, const std::string& db) { mcs->createTableBuilder(db, "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI) - .setVgid(1) + .setVgid(2) .addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .addColumn("c1", TSDB_DATA_TYPE_INT) .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) @@ -182,9 +183,9 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) { .addTag("tag2", TSDB_DATA_TYPE_BINARY, 20) .addTag("tag3", TSDB_DATA_TYPE_TIMESTAMP); builder.done(); - mcs->createSubTable(db, "st1", "st1s1", 1); - mcs->createSubTable(db, "st1", "st1s2", 2); - mcs->createSubTable(db, "st1", "st1s3", 1); + mcs->createSubTable(db, "st1", "st1s1", 2); + mcs->createSubTable(db, "st1", "st1s2", 3); + mcs->createSubTable(db, "st1", "st1s3", 2); } { ITableBuilder& builder = mcs->createTableBuilder(db, "st2", TSDB_SUPER_TABLE, 3, 1) @@ -194,8 +195,8 @@ void generateTestStables(MockCatalogService* mcs, const std::string& db) { .addColumn("c2", TSDB_DATA_TYPE_BINARY, 20) .addTag("jtag", TSDB_DATA_TYPE_JSON); builder.done(); - mcs->createSubTable(db, "st2", "st2s1", 1); - mcs->createSubTable(db, "st2", "st2s2", 2); + mcs->createSubTable(db, "st2", "st2s1", 2); + mcs->createSubTable(db, "st2", "st2s2", 3); } } @@ -247,6 +248,13 @@ int32_t __catalogGetCachedTableHashVgroup(SCatalog* pCtg, const SName* pTableNam return code; } +int32_t __catalogGetCachedTableVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { + int32_t code = g_mockCatalogService->catalogGetTableMeta(pTableName, pTableMeta, true); + if (code) return code; + code = g_mockCatalogService->catalogGetTableHashVgroup(pTableName, pVgroup, true); + return code; +} + int32_t __catalogGetTableDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pVgList) { return g_mockCatalogService->catalogGetTableDistVgInfo(pTableName, pVgList); @@ -315,6 +323,7 @@ void initMetaDataEnv() { stub.set(catalogGetCachedSTableMeta, __catalogGetCachedTableMeta); stub.set(catalogGetTableHashVgroup, __catalogGetTableHashVgroup); stub.set(catalogGetCachedTableHashVgroup, __catalogGetCachedTableHashVgroup); + stub.set(catalogGetCachedTableVgMeta, __catalogGetCachedTableVgMeta); stub.set(catalogGetTableDistVgInfo, __catalogGetTableDistVgInfo); stub.set(catalogGetDBVgVersion, __catalogGetDBVgVersion); stub.set(catalogGetDBVgList, __catalogGetDBVgList); diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index be2e4b90b9901c7241f5cb62f97408ea00829e6f..9cc55a7cd55a0060635e9a9044ac076ce5a77119 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -20,15 +20,19 @@ #include #include +#include "systable.h" +#include "tdatablock.h" +#include "tmisce.h" #include "tname.h" #include "ttypes.h" -#include "tmisce.h" + +using std::string; std::unique_ptr g_mockCatalogService; class TableBuilder : public ITableBuilder { public: - virtual TableBuilder& addColumn(const std::string& name, int8_t type, int32_t bytes) { + virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) { assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + (colId_ - 1); col->type = type; @@ -142,27 +146,16 @@ class MockCatalogServiceImpl { } int32_t catalogGetDBVgList(const char* pDbFName, SArray** pVgList) const { - std::string dbFName(pDbFName); - DbMetaCache::const_iterator it = meta_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1)); - if (meta_.end() == it) { - return TSDB_CODE_FAILED; - } - std::set vgSet; - *pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo)); - for (const auto& vgs : it->second) { - for (const auto& vg : vgs.second->vgs) { - if (0 == vgSet.count(vg.vgId)) { - taosArrayPush(*pVgList, &vg); - vgSet.insert(vg.vgId); - } - } + string dbName(string(pDbFName).substr(string(pDbFName).find_last_of('.') + 1)); + if (0 == dbName.compare(TSDB_INFORMATION_SCHEMA_DB) || 0 == dbName.compare(TSDB_PERFORMANCE_SCHEMA_DB)) { + return catalogGetAllDBVgList(pVgList); } - return TSDB_CODE_SUCCESS; + return catalogGetDBVgListImpl(dbName, pVgList); } int32_t catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pDbCfg) const { - std::string dbFName(pDbFName); - DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1)); + string dbFName(pDbFName); + DbCfgCache::const_iterator it = dbCfg_.find(dbFName.substr(string(pDbFName).find_last_of('.') + 1)); if (dbCfg_.end() == it) { return TSDB_CODE_FAILED; } @@ -171,7 +164,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { + int32_t catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const { auto it = udf_.find(funcName); if (udf_.end() == it) { return TSDB_CODE_FAILED; @@ -236,15 +229,15 @@ class MockCatalogServiceImpl { return code; } - TableBuilder& createTableBuilder(const std::string& db, const std::string& tbname, int8_t tableType, - int32_t numOfColumns, int32_t numOfTags) { + TableBuilder& createTableBuilder(const string& db, const string& tbname, int8_t tableType, int32_t numOfColumns, + int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); meta_[db][tbname]->schema->uid = getNextId(); return *(builder_.get()); } - void createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, int16_t vgid) { + void createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) { std::unique_ptr table; if (TSDB_CODE_SUCCESS != copyTableSchemaMeta(db, stbname, &table)) { throw std::runtime_error("copyTableSchemaMeta failed"); @@ -274,13 +267,13 @@ class MockCatalogServiceImpl { // string field length #define SFL 20 // string field header -#define SH(h) CA(SFL, std::string(h)) +#define SH(h) CA(SFL, string(h)) // string field #define SF(n) CA(SFL, n) // integer field length #define IFL 10 // integer field header -#define IH(i) CA(IFL, std::string(i)) +#define IH(i) CA(IFL, string(i)) // integer field #define IF(i) CA(IFL, std::to_string(i)) // split line @@ -308,7 +301,7 @@ class MockCatalogServiceImpl { int16_t numOfFields = numOfColumns + schema->tableInfo.numOfTags; for (int16_t i = 0; i < numOfFields; ++i) { const SSchema* col = schema->schema + i; - std::cout << SF(std::string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type)) + std::cout << SF(string(col->name)) << SH(ftToString(i, numOfColumns)) << SH(dtToString(col->type)) << IF(col->bytes) << std::endl; } std::cout << std::endl; @@ -316,7 +309,7 @@ class MockCatalogServiceImpl { } } - void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { + void createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { std::shared_ptr info(new SFuncInfo); strcpy(info->name, func.c_str()); info->funcType = funcType; @@ -342,19 +335,19 @@ class MockCatalogServiceImpl { info.expr = strdup(pReq->expr); auto it = index_.find(pReq->stb); if (index_.end() == it) { - index_.insert(std::make_pair(std::string(pReq->stb), std::vector{info})); + index_.insert(std::make_pair(string(pReq->stb), std::vector{info})); } else { it->second.push_back(info); } } - void createDnode(int32_t dnodeId, const std::string& host, int16_t port) { + void createDnode(int32_t dnodeId, const string& host, int16_t port) { SEpSet epSet = {0}; addEpIntoEpSet(&epSet, host.c_str(), port); dnode_.insert(std::make_pair(dnodeId, epSet)); } - void createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { + void createDatabase(const string& db, bool rollup, int8_t cacheLast) { SDbCfgInfo cfg = {0}; if (rollup) { cfg.pRetensions = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SRetention)); @@ -364,12 +357,12 @@ class MockCatalogServiceImpl { } private: - typedef std::map> TableMetaCache; - typedef std::map DbMetaCache; - typedef std::map> UdfMetaCache; - typedef std::map> IndexMetaCache; - typedef std::map DnodeCache; - typedef std::map DbCfgCache; + typedef std::map> TableMetaCache; + typedef std::map DbMetaCache; + typedef std::map> UdfMetaCache; + typedef std::map> IndexMetaCache; + typedef std::map DnodeCache; + typedef std::map DbCfgCache; uint64_t getNextId() { return id_++; } @@ -386,15 +379,15 @@ class MockCatalogServiceImpl { return pDst; } - std::string toDbname(const std::string& dbFullName) const { - std::string::size_type n = dbFullName.find("."); - if (n == std::string::npos) { + string toDbname(const string& dbFullName) const { + string::size_type n = dbFullName.find("."); + if (n == string::npos) { return dbFullName; } return dbFullName.substr(n + 1); } - std::string ttToString(int8_t tableType) const { + string ttToString(int8_t tableType) const { switch (tableType) { case TSDB_SUPER_TABLE: return "super table"; @@ -407,7 +400,7 @@ class MockCatalogServiceImpl { } } - std::string pToString(uint8_t precision) const { + string pToString(uint8_t precision) const { switch (precision) { case TSDB_TIME_PRECISION_MILLI: return "millisecond"; @@ -420,19 +413,18 @@ class MockCatalogServiceImpl { } } - std::string dtToString(int8_t type) const { return tDataTypes[type].name; } + string dtToString(int8_t type) const { return tDataTypes[type].name; } - std::string ftToString(int16_t colid, int16_t numOfColumns) const { + string ftToString(int16_t colid, int16_t numOfColumns) const { return (0 == colid ? "column" : (colid < numOfColumns ? "column" : "tag")); } - STableMeta* getTableSchemaMeta(const std::string& db, const std::string& tbname) const { + STableMeta* getTableSchemaMeta(const string& db, const string& tbname) const { std::shared_ptr table = getTableMeta(db, tbname); return table ? table->schema : nullptr; } - int32_t copyTableSchemaMeta(const std::string& db, const std::string& tbname, - std::unique_ptr* dst) const { + int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { return TSDB_CODE_TSC_INVALID_TABLE_NAME; @@ -440,13 +432,13 @@ class MockCatalogServiceImpl { int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); if (!dst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(dst->get(), src, len); return TSDB_CODE_SUCCESS; } - int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SVgroupInfo* vg) const { + int32_t copyTableVgroup(const string& db, const string& tbname, SVgroupInfo* vg) const { std::shared_ptr table = getTableMeta(db, tbname); if (table->vgs.empty()) { return TSDB_CODE_SUCCESS; @@ -455,7 +447,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - int32_t copyTableVgroup(const std::string& db, const std::string& tbname, SArray** vgList) const { + int32_t copyTableVgroup(const string& db, const string& tbname, SArray** vgList) const { std::shared_ptr table = getTableMeta(db, tbname); if (table->vgs.empty()) { return TSDB_CODE_SUCCESS; @@ -467,7 +459,7 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } - std::shared_ptr getTableMeta(const std::string& db, const std::string& tbname) const { + std::shared_ptr getTableMeta(const string& db, const string& tbname) const { DbMetaCache::const_iterator it = meta_.find(db); if (meta_.end() == it) { return std::shared_ptr(); @@ -527,6 +519,40 @@ class MockCatalogServiceImpl { return code; } + int32_t catalogGetDBVgListImpl(const string& dbName, SArray** pVgList) const { + DbMetaCache::const_iterator it = meta_.find(dbName); + if (meta_.end() == it) { + return TSDB_CODE_FAILED; + } + std::set vgSet; + *pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo)); + for (const auto& vgs : it->second) { + for (const auto& vg : vgs.second->vgs) { + if (0 == vgSet.count(vg.vgId)) { + taosArrayPush(*pVgList, &vg); + vgSet.insert(vg.vgId); + } + } + } + return TSDB_CODE_SUCCESS; + } + + int32_t catalogGetAllDBVgList(SArray** pVgList) const { + std::set vgSet; + *pVgList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVgroupInfo)); + for (const auto& db : meta_) { + for (const auto& vgs : db.second) { + for (const auto& vg : vgs.second->vgs) { + if (0 == vgSet.count(vg.vgId)) { + taosArrayPush(*pVgList, &vg); + vgSet.insert(vg.vgId); + } + } + } + } + return TSDB_CODE_SUCCESS; + } + int32_t getAllDbCfg(SArray* pDbCfgReq, SArray** pDbCfgData) const { int32_t code = TSDB_CODE_SUCCESS; if (NULL != pDbCfgReq) { @@ -634,30 +660,29 @@ MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) { MockCatalogService::~MockCatalogService() {} -ITableBuilder& MockCatalogService::createTableBuilder(const std::string& db, const std::string& tbname, - int8_t tableType, int32_t numOfColumns, int32_t numOfTags) { +ITableBuilder& MockCatalogService::createTableBuilder(const string& db, const string& tbname, int8_t tableType, + int32_t numOfColumns, int32_t numOfTags) { return impl_->createTableBuilder(db, tbname, tableType, numOfColumns, numOfTags); } -void MockCatalogService::createSubTable(const std::string& db, const std::string& stbname, const std::string& tbname, - int16_t vgid) { +void MockCatalogService::createSubTable(const string& db, const string& stbname, const string& tbname, int16_t vgid) { impl_->createSubTable(db, stbname, tbname, vgid); } void MockCatalogService::showTables() const { impl_->showTables(); } -void MockCatalogService::createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, +void MockCatalogService::createFunction(const string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize) { impl_->createFunction(func, funcType, outputType, outputLen, bufSize); } void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } -void MockCatalogService::createDnode(int32_t dnodeId, const std::string& host, int16_t port) { +void MockCatalogService::createDnode(int32_t dnodeId, const string& host, int16_t port) { impl_->createDnode(dnodeId, host, port); } -void MockCatalogService::createDatabase(const std::string& db, bool rollup, int8_t cacheLast) { +void MockCatalogService::createDatabase(const string& db, bool rollup, int8_t cacheLast) { impl_->createDatabase(db, rollup, cacheLast); } @@ -683,7 +708,7 @@ int32_t MockCatalogService::catalogGetDBCfg(const char* pDbFName, SDbCfgInfo* pD return impl_->catalogGetDBCfg(pDbFName, pDbCfg); } -int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { +int32_t MockCatalogService::catalogGetUdfInfo(const string& funcName, SFuncInfo* pInfo) const { return impl_->catalogGetUdfInfo(funcName, pInfo); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 634fd3e1275cbe094920bb7958523dc8b8ad1d76..17d02c1cce7084c99d025d89b36c69e223083069 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -48,7 +48,7 @@ TEST_F(ParserInitialCTest, createAccount) { * | PRECISION {'ms' | 'us' | 'ns'} * | REPLICA value * | RETENTIONS ingestion_duration:keep_duration ... - * | STRICT {'off' | 'on'} + * | STRICT {'off' | 'on'} // not support * | WAL_LEVEL value * | VGROUPS value * | SINGLE_STABLE {0 | 1} @@ -216,7 +216,7 @@ TEST_F(ParserInitialCTest, createDatabase) { addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - setDbStrictaFunc(1); + // setDbStrictaFunc(1); setDbWalLevelFunc(2); setDbVgroupsFunc(100); setDbSingleStableFunc(1); @@ -244,7 +244,7 @@ TEST_F(ParserInitialCTest, createDatabase) { "PRECISION 'ns' " "REPLICA 3 " "RETENTIONS 15s:7d,1m:21d,15m:500d " - "STRICT 'on' " + // "STRICT 'on' " "WAL_LEVEL 2 " "VGROUPS 100 " "SINGLE_STABLE 1 " @@ -375,9 +375,68 @@ TEST_F(ParserInitialCTest, createFunction) { TEST_F(ParserInitialCTest, createSmaIndex) { useDb("root", "test"); + SMCreateSmaReq expect = {0}; + + auto setCreateSmacReq = [&](const char* pIndexName, const char* pStbName, int64_t interval, int8_t intervalUnit, + int64_t offset = 0, int64_t sliding = -1, int8_t slidingUnit = -1, int8_t igExists = 0) { + memset(&expect, 0, sizeof(SMCreateSmaReq)); + strcpy(expect.name, pIndexName); + strcpy(expect.stb, pStbName); + expect.igExists = igExists; + expect.intervalUnit = intervalUnit; + expect.slidingUnit = slidingUnit < 0 ? intervalUnit : slidingUnit; + expect.timezone = 0; + expect.dstVgId = 1; + expect.interval = interval; + expect.offset = offset; + expect.sliding = sliding < 0 ? interval : sliding; + expect.maxDelay = -1; + expect.watermark = TSDB_DEFAULT_ROLLUP_WATERMARK; + expect.deleteMark = TSDB_DEFAULT_ROLLUP_DELETE_MARK; + }; + + auto setOptionsForCreateSmacReq = [&](int64_t maxDelay, int64_t watermark, int64_t deleteMark) { + expect.maxDelay = maxDelay; + expect.watermark = watermark; + expect.deleteMark = deleteMark; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_INDEX_STMT); + SMCreateSmaReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(std::string(req.stb), std::string(expect.stb)); + ASSERT_EQ(req.igExists, expect.igExists); + ASSERT_EQ(req.intervalUnit, expect.intervalUnit); + ASSERT_EQ(req.slidingUnit, expect.slidingUnit); + ASSERT_EQ(req.timezone, expect.timezone); + ASSERT_EQ(req.dstVgId, expect.dstVgId); + ASSERT_EQ(req.interval, expect.interval); + ASSERT_EQ(req.offset, expect.offset); + ASSERT_EQ(req.sliding, expect.sliding); + ASSERT_EQ(req.maxDelay, expect.maxDelay); + ASSERT_EQ(req.watermark, expect.watermark); + ASSERT_EQ(req.deleteMark, expect.deleteMark); + ASSERT_GT(req.exprLen, 0); + ASSERT_EQ(req.tagsFilterLen, 0); + ASSERT_GT(req.sqlLen, 0); + ASSERT_GT(req.astLen, 0); + ASSERT_NE(req.expr, nullptr); + ASSERT_EQ(req.tagsFilter, nullptr); + ASSERT_NE(req.sql, nullptr); + ASSERT_NE(req.ast, nullptr); + tFreeSMCreateSmaReq(&req); + }); + + setCreateSmacReq("0.test.index1", "0.test.t1", 10 * MILLISECOND_PER_SECOND, 's'); run("CREATE SMA INDEX index1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); - run("CREATE SMA INDEX index2 ON st1 FUNCTION(MAX(c1), MIN(tag1)) INTERVAL(10s)"); + setCreateSmacReq("0.test.index2", "0.test.st1", 5 * MILLISECOND_PER_SECOND, 's'); + setOptionsForCreateSmacReq(10 * MILLISECOND_PER_SECOND, 20 * MILLISECOND_PER_SECOND, 1000 * MILLISECOND_PER_SECOND); + run("CREATE SMA INDEX index2 ON st1 FUNCTION(MAX(c1), MIN(tag1)) INTERVAL(5s) WATERMARK 20s MAX_DELAY 10s " + "DELETE_MARK 1000s"); } TEST_F(ParserInitialCTest, createMnode) { @@ -408,23 +467,26 @@ TEST_F(ParserInitialCTest, createStable) { memset(&expect, 0, sizeof(SMCreateStbReq)); }; - auto setCreateStbReqFunc = [&](const char* pDbName, const char* pTbName, int8_t igExists = 0, int64_t delay1 = -1, - int64_t delay2 = -1, int64_t watermark1 = TSDB_DEFAULT_ROLLUP_WATERMARK, - int64_t watermark2 = TSDB_DEFAULT_ROLLUP_WATERMARK, - int32_t ttl = TSDB_DEFAULT_TABLE_TTL, const char* pComment = nullptr) { - int32_t len = snprintf(expect.name, sizeof(expect.name), "0.%s.%s", pDbName, pTbName); - expect.name[len] = '\0'; - expect.igExists = igExists; - expect.delay1 = delay1; - expect.delay2 = delay2; - expect.watermark1 = watermark1; - expect.watermark2 = watermark2; - // expect.ttl = ttl; - if (nullptr != pComment) { - expect.pComment = strdup(pComment); - expect.commentLen = strlen(pComment); - } - }; + auto setCreateStbReqFunc = + [&](const char* pDbName, const char* pTbName, int8_t igExists = 0, int64_t delay1 = -1, int64_t delay2 = -1, + int64_t watermark1 = TSDB_DEFAULT_ROLLUP_WATERMARK, int64_t watermark2 = TSDB_DEFAULT_ROLLUP_WATERMARK, + int64_t deleteMark1 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, int64_t deleteMark2 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, + int32_t ttl = TSDB_DEFAULT_TABLE_TTL, const char* pComment = nullptr) { + int32_t len = snprintf(expect.name, sizeof(expect.name), "0.%s.%s", pDbName, pTbName); + expect.name[len] = '\0'; + expect.igExists = igExists; + expect.delay1 = delay1; + expect.delay2 = delay2; + expect.watermark1 = watermark1; + expect.watermark2 = watermark2; + expect.deleteMark1 = deleteMark1; + expect.deleteMark2 = deleteMark2; + // expect.ttl = ttl; + if (nullptr != pComment) { + expect.pComment = strdup(pComment); + expect.commentLen = strlen(pComment); + } + }; auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, int8_t flags = COL_SMA_ON) { @@ -511,7 +573,8 @@ TEST_F(ParserInitialCTest, createStable) { clearCreateStbReq(); setCreateStbReqFunc("rollup_db", "t1", 1, 100 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_MINUTE, 10, - 1 * MILLISECOND_PER_MINUTE, 100, "test create table"); + 1 * MILLISECOND_PER_MINUTE, 1000 * MILLISECOND_PER_SECOND, 200 * MILLISECOND_PER_MINUTE, 100, + "test create table"); addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); addFieldToCreateStbReqFunc(true, "c2", TSDB_DATA_TYPE_UINT); @@ -549,7 +612,8 @@ TEST_F(ParserInitialCTest, createStable) { "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " - "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m"); + "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " + "DELETE_MARK 1000s,200m"); clearCreateStbReq(); } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e0a67c32e5ba4b16e056839f9efe53fe526a386c..205c70e0dffdcb77b333f310da93554a782a251b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -250,11 +250,12 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols } if (NULL == pScanCols) { - return NULL == pScanPseudoCols - ? SCAN_TYPE_TABLE - : ((FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType) - ? SCAN_TYPE_BLOCK_INFO - : SCAN_TYPE_TABLE); + if (NULL == pScanPseudoCols) { + return SCAN_TYPE_TABLE; + } + return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType + ? SCAN_TYPE_BLOCK_INFO + : SCAN_TYPE_TABLE; } return SCAN_TYPE_TABLE; @@ -333,6 +334,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT return TSDB_CODE_SUCCESS; } +static bool needScanDefaultCol(EScanType scanType) { return SCAN_TYPE_TABLE_COUNT != scanType; } + static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = NULL; @@ -367,7 +370,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->hasNormalCols = true; } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && needScanDefaultCol(pScan->scanType)) { code = addDefaultScanCol(pRealTable->pMeta, &pScan->pScanCols); } @@ -699,6 +702,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm if (pCxt->pPlanCxt->streamQuery) { pWindow->triggerType = pCxt->pPlanCxt->triggerType; pWindow->watermark = pCxt->pPlanCxt->watermark; + pWindow->deleteMark = pCxt->pPlanCxt->deleteMark; pWindow->igExpired = pCxt->pPlanCxt->igExpired; } pWindow->inputTsOrder = ORDER_ASC; @@ -810,6 +814,29 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } +static int32_t createWindowLogicNodeByEvent(SLogicPlanContext* pCxt, SEventWindowNode* pEvent, SSelectStmt* pSelect, + SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_EVENT; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; + pWindow->pStartCond = nodesCloneNode(pEvent->pStartCond); + pWindow->pEndCond = nodesCloneNode(pEvent->pEndCond); + pWindow->pTspk = nodesCloneNode(pEvent->pCol); + if (NULL == pWindow->pStartCond || NULL == pWindow->pEndCond || NULL == pWindow->pTspk) { + nodesDestroyNode((SNode*)pWindow); + return TSDB_CODE_OUT_OF_MEMORY; + } + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; @@ -822,6 +849,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); + case QUERY_NODE_EVENT_WINDOW: + return createWindowLogicNodeByEvent(pCxt, (SEventWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index b8b6e444129cf92864cfcfef64e44f845027e0b2..e1687fc3a5e3136fb34893c2af6994c4c4893586 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -16,6 +16,7 @@ #include "filter.h" #include "functionMgt.h" #include "planInt.h" +#include "systable.h" #include "tglobal.h" #include "ttime.h" @@ -64,6 +65,7 @@ typedef enum ECondAction { } ECondAction; typedef bool (*FMayBeOptimized)(SLogicNode* pNode); +typedef bool (*FShouldBeOptimized)(SLogicNode* pNode, void* pInfo); static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) { if (func(pNode)) { @@ -79,6 +81,19 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) return NULL; } +static bool optFindEligibleNode(SLogicNode* pNode, FShouldBeOptimized func, void* pInfo) { + if (func(pNode, pInfo)) { + return true; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + if (optFindEligibleNode((SLogicNode*)pChild, func, pInfo)) { + return true; + } + } + return false; +} + static void optResetParent(SLogicNode* pNode) { SNode* pChild = NULL; FOREACH(pChild, pNode->pChildren) { ((SLogicNode*)pChild)->pParent = pNode; } @@ -315,6 +330,7 @@ static void scanPathOptSetScanWin(SScanLogicNode* pScan) { pScan->slidingUnit = ((SWindowLogicNode*)pParent)->slidingUnit; pScan->triggerType = ((SWindowLogicNode*)pParent)->triggerType; pScan->watermark = ((SWindowLogicNode*)pParent)->watermark; + pScan->deleteMark = ((SWindowLogicNode*)pParent)->deleteMark; pScan->igExpired = ((SWindowLogicNode*)pParent)->igExpired; } } @@ -2454,6 +2470,243 @@ static int32_t pushDownLimitOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return TSDB_CODE_SUCCESS; } +typedef struct STbCntScanOptInfo { + SAggLogicNode* pAgg; + SScanLogicNode* pScan; + SName table; +} STbCntScanOptInfo; + +static bool tbCntScanOptIsEligibleGroupKeys(SNodeList* pGroupKeys) { + if (NULL == pGroupKeys) { + return true; + } + + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pGroupKeys) { + SNode* pKey = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + if (QUERY_NODE_COLUMN != nodeType(pKey)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pKey; + if (0 != strcmp(pCol->colName, "db_name") && 0 != strcmp(pCol->colName, "stable_name")) { + return false; + } + } + + return true; +} + +static bool tbCntScanOptNotNullableExpr(SNode* pNode) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + const char* pColName = ((SColumnNode*)pNode)->colName; + return 0 == strcmp(pColName, "*") || 0 == strcmp(pColName, "db_name") || 0 == strcmp(pColName, "stable_name") || + 0 == strcmp(pColName, "table_name"); +} + +static bool tbCntScanOptIsEligibleAggFuncs(SNodeList* pAggFuncs) { + SNode* pNode = NULL; + FOREACH(pNode, pAggFuncs) { + SFunctionNode* pFunc = (SFunctionNode*)nodesListGetNode(pAggFuncs, 0); + if (FUNCTION_TYPE_COUNT != pFunc->funcType || + !tbCntScanOptNotNullableExpr(nodesListGetNode(pFunc->pParameterList, 0))) { + return false; + } + } + return true; +} + +static bool tbCntScanOptIsEligibleAgg(SAggLogicNode* pAgg) { + return tbCntScanOptIsEligibleGroupKeys(pAgg->pGroupKeys) && tbCntScanOptIsEligibleAggFuncs(pAgg->pAggFuncs); +} + +static bool tbCntScanOptGetColValFromCond(SOperatorNode* pOper, SColumnNode** pCol, SValueNode** pVal) { + if (OP_TYPE_EQUAL != pOper->opType) { + return false; + } + + *pCol = NULL; + *pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + *pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + *pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + *pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + *pVal = (SValueNode*)pOper->pRight; + } + + return NULL != *pCol && NULL != *pVal; +} + +static bool tbCntScanOptIsEligibleLogicCond(STbCntScanOptInfo* pInfo, SLogicConditionNode* pCond) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return false; + } + + bool hasDbCond = false; + bool hasStbCond = false; + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { + if (QUERY_NODE_OPERATOR != nodeType(pNode) || !tbCntScanOptGetColValFromCond((SOperatorNode*)pNode, &pCol, &pVal)) { + return false; + } + if (!hasDbCond && 0 == strcmp(pCol->colName, "db_name")) { + hasDbCond = true; + strcpy(pInfo->table.dbname, pVal->literal); + } else if (!hasStbCond && 0 == strcmp(pCol->colName, "stable_name")) { + hasStbCond = true; + strcpy(pInfo->table.tname, pVal->literal); + } else { + return false; + } + } + return hasDbCond; +} + +static bool tbCntScanOptIsEligibleOpCond(SOperatorNode* pCond) { + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (!tbCntScanOptGetColValFromCond(pCond, &pCol, &pVal)) { + return false; + } + return 0 == strcmp(pCol->colName, "db_name"); +} + +static bool tbCntScanOptIsEligibleConds(STbCntScanOptInfo* pInfo, SNode* pConditions) { + if (NULL == pConditions) { + return true; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pConditions)) { + return tbCntScanOptIsEligibleLogicCond(pInfo, (SLogicConditionNode*)pConditions); + } + + if (QUERY_NODE_OPERATOR == nodeType(pConditions)) { + return tbCntScanOptIsEligibleOpCond((SOperatorNode*)pConditions); + } + + return false; +} + +static bool tbCntScanOptIsEligibleScan(STbCntScanOptInfo* pInfo) { + if (0 != strcmp(pInfo->pScan->tableName.dbname, TSDB_INFORMATION_SCHEMA_DB) || + 0 != strcmp(pInfo->pScan->tableName.tname, TSDB_INS_TABLE_TABLES) || NULL != pInfo->pScan->pGroupTags) { + return false; + } + if (1 == pInfo->pScan->pVgroupList->numOfVgroups && MNODE_HANDLE == pInfo->pScan->pVgroupList->vgroups[0].vgId) { + return false; + } + return tbCntScanOptIsEligibleConds(pInfo, pInfo->pScan->node.pConditions); +} + +static bool tbCntScanOptShouldBeOptimized(SLogicNode* pNode, STbCntScanOptInfo* pInfo) { + if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + return false; + } + + pInfo->pAgg = (SAggLogicNode*)pNode; + pInfo->pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); + return tbCntScanOptIsEligibleAgg(pInfo->pAgg) && tbCntScanOptIsEligibleScan(pInfo); +} + +static SNode* tbCntScanOptCreateTableCountFunc() { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return NULL; + } + strcpy(pFunc->functionName, "_table_count"); + strcpy(pFunc->node.aliasName, "_table_count"); + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc, NULL, 0)) { + nodesDestroyNode((SNode*)pFunc); + return NULL; + } + return (SNode*)pFunc; +} + +static int32_t tbCntScanOptRewriteScan(STbCntScanOptInfo* pInfo) { + pInfo->pScan->scanType = SCAN_TYPE_TABLE_COUNT; + strcpy(pInfo->pScan->tableName.dbname, pInfo->table.dbname); + strcpy(pInfo->pScan->tableName.tname, pInfo->table.tname); + NODES_DESTORY_LIST(pInfo->pScan->node.pTargets); + NODES_DESTORY_LIST(pInfo->pScan->pScanCols); + NODES_DESTORY_NODE(pInfo->pScan->node.pConditions); + NODES_DESTORY_LIST(pInfo->pScan->pScanPseudoCols); + int32_t code = nodesListMakeStrictAppend(&pInfo->pScan->pScanPseudoCols, tbCntScanOptCreateTableCountFunc()); + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExpr(nodesListGetNode(pInfo->pScan->pScanPseudoCols, 0), &pInfo->pScan->node.pTargets); + } + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, pInfo->pAgg->pGroupKeys) { + SNode* pGroupCol = nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0); + code = nodesListMakeStrictAppend(&pInfo->pScan->pGroupTags, nodesCloneNode(pGroupCol)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pScan->pScanCols, nodesCloneNode(pGroupCol)); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pScan->node.pTargets, nodesCloneNode(pGroupCol)); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t tbCntScanOptCreateSumFunc(SFunctionNode* pCntFunc, SNode* pParam, SNode** pOutput) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "sum"); + strcpy(pFunc->node.aliasName, pCntFunc->node.aliasName); + int32_t code = createColumnByRewriteExpr(pParam, &pFunc->pParameterList); + if (TSDB_CODE_SUCCESS == code) { + code = fmGetFuncInfo(pFunc, NULL, 0); + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = (SNode*)pFunc; + } else { + nodesDestroyNode((SNode*)pFunc); + } + return code; +} + +static int32_t tbCntScanOptRewriteAgg(SAggLogicNode* pAgg) { + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0); + SNode* pSum = NULL; + int32_t code = tbCntScanOptCreateSumFunc((SFunctionNode*)nodesListGetNode(pAgg->pAggFuncs, 0), + nodesListGetNode(pScan->pScanPseudoCols, 0), &pSum); + if (TSDB_CODE_SUCCESS == code) { + NODES_DESTORY_LIST(pAgg->pAggFuncs); + code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, pSum); + } + if (TSDB_CODE_SUCCESS == code) { + code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, 0, pAgg); + } + NODES_DESTORY_LIST(pAgg->pGroupKeys); + return code; +} + +static int32_t tableCountScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + STbCntScanOptInfo info = {0}; + if (!optFindEligibleNode(pLogicSubplan->pNode, (FShouldBeOptimized)tbCntScanOptShouldBeOptimized, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = tbCntScanOptRewriteScan(&info); + if (TSDB_CODE_SUCCESS == code) { + code = tbCntScanOptRewriteAgg(info.pAgg); + } + return code; +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ScanPath", .optimizeFunc = scanPathOptimize}, @@ -2468,7 +2721,8 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}, {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize}, {.pName = "TagScan", .optimizeFunc = tagScanOptimize}, - {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize} + {.pName = "PushDownLimit", .optimizeFunc = pushDownLimitOptimize}, + {.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize}, }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 379bfe90c8c3e4a65beee8d1d66dce1aebcc325f..78ae3c1c3b9bbf68066e53f32385f4c726a2d94d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -489,6 +489,8 @@ static ENodeType getScanOperatorType(EScanType scanType) { return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; case SCAN_TYPE_BLOCK_INFO: return QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; + case SCAN_TYPE_TABLE_COUNT: + return QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN; default: break; } @@ -524,6 +526,24 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu pScan->ignoreNull = pScanLogicNode->igLastNull; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); + return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); +} + +static int32_t createTableCountScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, + SScanLogicNode* pScanLogicNode, SPhysiNode** pPhyNode) { + STableCountScanPhysiNode* pScan = (STableCountScanPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pScanLogicNode, + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN); + if (NULL == pScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); + if (NULL != pScanLogicNode->pGroupTags && NULL == pScan->pGroupTags) { + nodesDestroyNode((SNode*)pScan); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pScan->groupSort = pScanLogicNode->groupSort; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); @@ -589,7 +609,6 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; pScan->sysInfo = pCxt->pPlanCxt->sysInfo; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLE_DISTRIBUTED) || 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { @@ -624,6 +643,8 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); + case SCAN_TYPE_TABLE_COUNT: + return createTableCountScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_LAST_ROW: return createLastRowScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: @@ -1118,6 +1139,7 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* SWindowLogicNode* pWindowLogicNode) { pWindow->triggerType = pWindowLogicNode->triggerType; pWindow->watermark = pWindowLogicNode->watermark; + pWindow->deleteMark = pWindowLogicNode->deleteMark; pWindow->igExpired = pWindowLogicNode->igExpired; pWindow->inputTsOrder = pWindowLogicNode->inputTsOrder; pWindow->outputTsOrder = pWindowLogicNode->outputTsOrder; @@ -1275,6 +1297,33 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return code; } +static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, + SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT : QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT)); + if (NULL == pEvent) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pEndCond, &pEvent->pEndCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = createWindowPhysiNodeFinalize(pCxt, pChildren, &pEvent->window, pWindowLogicNode); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pEvent; + } else { + nodesDestroyNode((SNode*)pEvent); + } + + return code; +} + static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { @@ -1284,6 +1333,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); + case WINDOW_TYPE_EVENT: + return createEventWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); default: break; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index bcf4b40e698ad8c6f9fde793af5eb538e44d9b6d..f6b1babf9501e12122bdcf97e8260251c348a617 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -292,6 +292,20 @@ static bool stbSplNeedSplitJoin(bool streamQuery, SJoinLogicNode* pJoin) { return true; } +static bool stbSplIsTableCountQuery(SLogicNode* pNode) { + if (1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pNode->pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pChild)) { + if (1 != LIST_LENGTH(((SLogicNode*)pChild)->pChildren)) { + return false; + } + pChild = nodesListGetNode(((SLogicNode*)pChild)->pChildren, 0); + } + return QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && SCAN_TYPE_TABLE_COUNT == ((SScanLogicNode*)pChild)->scanType; +} + static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { return ((SScanLogicNode*)pNode)->pGroupTags; @@ -340,7 +354,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { case QUERY_NODE_LOGIC_PLAN_AGG: return (!stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) || stbSplIsPartTableAgg((SAggLogicNode*)pNode)) && - stbSplHasMultiTbScan(streamQuery, pNode); + stbSplHasMultiTbScan(streamQuery, pNode) && !stbSplIsTableCountQuery(pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: return stbSplNeedSplitWindow(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -715,6 +729,18 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } +static int32_t stbSplSplitEventForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitEventForStream(pCxt, pInfo); + } else { + return stbSplSplitSessionOrStateForBatch(pCxt, pInfo); + } +} + static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } @@ -727,6 +753,8 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI return stbSplSplitSession(pCxt, pInfo); case WINDOW_TYPE_STATE: return stbSplSplitState(pCxt, pInfo); + case WINDOW_TYPE_EVENT: + return stbSplSplitEvent(pCxt, pInfo); default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index a13e959a369ff03141e7c1a612529defdb1b5321..72931413ccfccd05cb657be8555cc510832d8fb2 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -197,6 +197,15 @@ static int32_t adjustStateDataRequirement(SWindowLogicNode* pWindow, EDataOrderL return TSDB_CODE_SUCCESS; } +static int32_t adjustEventDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + if (requirement <= pWindow->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pWindow->node.resultDataOrder = requirement; + pWindow->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { switch (pWindow->winType) { case WINDOW_TYPE_INTERVAL: @@ -205,6 +214,8 @@ static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrder return adjustSessionDataRequirement(pWindow, requirement); case WINDOW_TYPE_STATE: return adjustStateDataRequirement(pWindow, requirement); + case WINDOW_TYPE_EVENT: + return adjustEventDataRequirement(pWindow, requirement); default: break; } diff --git a/source/libs/planner/test/planEventTest.cpp b/source/libs/planner/test/planEventTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c4db1459982de3c8daf6b8b37795a806ddd77d01 --- /dev/null +++ b/source/libs/planner/test/planEventTest.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanEventTest : public PlannerTestBase {}; + +TEST_F(PlanEventTest, basic) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); +} + +TEST_F(PlanEventTest, stable) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM st1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); +} diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 3ed23d6cb27f2044943ba4464199283ba09d8910..4741d241b5ac8b5541b6f92d08dbe1dd26151099 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -51,7 +51,7 @@ TEST_F(PlanOtherTest, createStreamUseSTable) { TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); - run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); + run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s) DELETE_MARK 1000s"); run("SELECT SUM(c4) FROM t1 INTERVAL(10s)"); diff --git a/source/libs/planner/test/planSysTbTest.cpp b/source/libs/planner/test/planSysTbTest.cpp index 6b40e381cc18cb75cc9271352cd654d31a74242b..fc00285a6d046b0ac70843d72e7a3203d0a9d6b8 100644 --- a/source/libs/planner/test/planSysTbTest.cpp +++ b/source/libs/planner/test/planSysTbTest.cpp @@ -20,13 +20,6 @@ using namespace std; class PlanSysTableTest : public PlannerTestBase {}; -TEST_F(PlanSysTableTest, show) { - useDb("root", "test"); - - run("show tables"); - run("show stables"); -} - TEST_F(PlanSysTableTest, informationSchema) { useDb("root", "information_schema"); @@ -38,3 +31,17 @@ TEST_F(PlanSysTableTest, withAgg) { run("SELECT COUNT(1) FROM ins_users"); } + +TEST_F(PlanSysTableTest, tableCount) { + useDb("root", "information_schema"); + + run("SELECT COUNT(*) FROM ins_tables"); + + run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test'"); + + run("SELECT COUNT(*) FROM ins_tables WHERE db_name = 'test' AND stable_name = 'st1'"); + + run("SELECT db_name, COUNT(*) FROM ins_tables GROUP BY db_name"); + + run("SELECT db_name, stable_name, COUNT(*) FROM ins_tables GROUP BY db_name, stable_name"); +} diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index b3620cedfb120bc25304f2d3c53cbac117b9adac..d89e669a90230894c5a7955a2eece9b1d2cc84e3 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -444,6 +444,7 @@ class PlannerTestBaseImpl { tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); + pCxt->deleteMark = req.deleteMark; tFreeSMCreateSmaReq(&req); nodesDestroyNode(pQuery->pRoot); pQuery->pRoot = pCxt->pAstRoot; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 55c47f9e55a46e047a1a4d5765eb3ef08a14cf9f..040ccc1694308d952ec43f57390562ab43e9b50b 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -159,7 +159,7 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra if (NULL == pMsg) { qError("0x%" PRIx64 " msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); destroySendMsgInfo(pInfo); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } @@ -244,6 +244,7 @@ void destroyQueryExecRes(SExecResult* pRes) { } case TDMT_VND_SUBMIT: { tDestroySSubmitRsp2((SSubmitRsp2*)pRes->res, TSDB_MSG_FLG_DECODE); + taosMemoryFreeClear(pRes->res); break; } case TDMT_SCH_QUERY: @@ -441,12 +442,23 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { int32_t metaSize = sizeof(STableMeta) + numOfField * sizeof(SSchema); *pDst = taosMemoryMalloc(metaSize); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, metaSize); return TSDB_CODE_SUCCESS; } +void freeVgInfo(SDBVgInfo* vgInfo) { + if (NULL == vgInfo) { + return; + } + + taosHashCleanup(vgInfo->vgHash); + taosArrayDestroy(vgInfo->vgArray); + + taosMemoryFreeClear(vgInfo); +} + int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -455,7 +467,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, sizeof(*pSrc)); if (pSrc->vgHash) { @@ -463,7 +475,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { taosMemoryFreeClear(*pDst); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } SVgroupInfo* vgInfo = NULL; @@ -475,8 +487,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) { qError("taosHashPut failed, vgId:%d", vgInfo->vgId); taosHashCancelIterate(pSrc->vgHash, pIter); - taosHashCleanup((*pDst)->vgHash); - taosMemoryFreeClear(*pDst); + freeVgInfo(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } @@ -486,3 +497,53 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pDst)->flags = pSrc->flags; + if (pSrc->name) { + (*pDst)->name = strdup(pSrc->name); + } + (*pDst)->uid = pSrc->uid; + (*pDst)->ctime = pSrc->ctime; + (*pDst)->ttl = pSrc->ttl; + (*pDst)->commentLen = pSrc->commentLen; + if (pSrc->comment) { + (*pDst)->comment = strdup(pSrc->comment); + } + (*pDst)->type = pSrc->type; + + if (pSrc->type == TSDB_CHILD_TABLE) { + if (pSrc->ctb.stbName) { + (*pDst)->ctb.stbName = strdup(pSrc->ctb.stbName); + } + (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; + (*pDst)->ctb.suid = pSrc->ctb.suid; + if (pSrc->ctb.tagName) { + (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + } + STag* pTag = (STag*)pSrc->ctb.pTag; + if (pTag) { + (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + memcpy((*pDst)->ctb.pTag, pTag, pTag->len); + } + } else { + (*pDst)->ntb.schemaRow.nCols = pSrc->ntb.schemaRow.nCols; + (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; + if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { + (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index fadc39f21d1be3c8d3df4a7d5af05064216fec27..2e4b000761220e4d49a3c03778efe89effebed8a 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -34,7 +34,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup = taosMemoryCalloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; @@ -52,7 +52,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->vgHash = taosHashInit(usedbRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } for (int32_t i = 0; i < usedbRsp->vgNum; ++i) { @@ -61,7 +61,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { qDebug("the %dth vgroup, id %d, epNum %d, current %s port %d", i, pVgInfo->vgId, pVgInfo->epSet.numOfEps, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].fqdn, pVgInfo->epSet.eps[pVgInfo->epSet.inUse].port); if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } } @@ -387,7 +387,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * STableMeta *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } pTableMeta->vgId = isStb ? 0 : msg->vgId; diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index af361323a7731f13512af85139d3ccdfd955ebf0..eb6091d60583ecbd07fbb202870d97dce4577d91 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -31,7 +31,7 @@ extern "C" { #define QW_DEFAULT_SCHEDULER_NUMBER 100 #define QW_DEFAULT_TASK_NUMBER 10000 -#define QW_DEFAULT_SCH_TASK_NUMBER 10000 +#define QW_DEFAULT_SCH_TASK_NUMBER 500 #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_SCH_TIMEOUT_MSEC 180000 @@ -247,7 +247,7 @@ typedef struct SQWorkerMgmt { #define QW_ERR_RET(c) \ do { \ - int32_t _code = (c); \ + int32_t _code = (c); \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ return _code; \ @@ -255,7 +255,7 @@ typedef struct SQWorkerMgmt { } while (0) #define QW_RET(c) \ do { \ - int32_t _code = (c); \ + int32_t _code = (c); \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ } \ @@ -263,7 +263,7 @@ typedef struct SQWorkerMgmt { } while (0) #define QW_ERR_JRET(c) \ do { \ - code = (c); \ + code = (c); \ if (code != TSDB_CODE_SUCCESS) { \ terrno = code; \ goto _return; \ diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index 4c4a41df8282e57023798edecbaec4beefb10691..db6e5b19fb51cda2a5bce77ed169b9ff2e69b05b 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -28,57 +28,57 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, return TSDB_CODE_SUCCESS; } - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_INIT) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: if (newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_FAIL) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_DROP: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_PART_SUCC) { - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } break; default: QW_TASK_ELOG("invalid task origStatus:%s", jobTaskStatusStr(oriStatus)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -214,20 +214,20 @@ void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { epSet.eps[2].port = 7300; ctx->phase = QW_PHASE_POST_QUERY; - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, &epSet); *rsped = true; return; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { - qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_SYN_NOT_LEADER, NULL); *rsped = true; return; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index d9a7cea41149325d954b71ad9d51661451fa2718..1a3a740b34918c695dfb79848f8df2c3500b0f3a 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -17,7 +17,7 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r (SRetrieveTableRsp *)(rpcMalloc ? rpcReallocCont(*rsp, msgSize) : taosMemoryRealloc(*rsp, msgSize)); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (NULL == *rsp) { @@ -37,7 +37,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->precision = input->precision; rsp->compressed = input->compressed; rsp->compLen = htonl(len); - rsp->numOfRows = htonl(input->numOfRows); + rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); rsp->numOfBlocks = htonl(input->numOfBlocks); } @@ -78,18 +78,18 @@ int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t c int32_t msgSize = tSerializeSQueryTableRsp(NULL, 0, &rsp); if (msgSize < 0) { qError("tSerializeSQueryTableRsp failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } void *pRsp = rpcMallocCont(msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSQueryTableRsp(pRsp, msgSize, &rsp) < 0) { qError("tSerializeSQueryTableRsp %d failed", msgSize); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg rpcRsp = { @@ -212,19 +212,19 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg pNewMsg = { @@ -250,7 +250,7 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } req->header.vgId = mgmt->nodeId; @@ -291,19 +291,19 @@ int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { QW_SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); rpcFreeCont(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { @@ -327,17 +327,17 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo * int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = rpcMallocCont(msgSize); if (NULL == msg) { QW_SCH_ELOG("calloc %d failed", msgSize); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { QW_SCH_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - taosMemoryFree(msg); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + rpcFreeCont(msg); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SRpcMsg brokenMsg = { diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 86fd1d533cb251904428d696b31bfb1e4716fe77..fdd2775daacbbadecd600f117ea3928c162e8f08 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -74,9 +74,11 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + newSch.hbBrokenTs = taosGetTimestampMs(); + if (NULL == newSch.tasksHash) { QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } QW_LOCK(QW_WRITE, &mgmt->schLock); @@ -87,7 +89,7 @@ int32_t qwAddSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType) { QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashCleanup(newSch.tasksHash); @@ -114,7 +116,7 @@ int32_t qwAcquireSchedulerImpl(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQW QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); } else { QW_SCH_ELOG("unknown notExistOpt:%d", nOpt); - QW_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_RET(TSDB_CODE_APP_ERROR); } } @@ -171,7 +173,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i } } else { QW_TASK_ELOG("taosHashPut to tasksHash failed, error:%x - %s", code, tstrerror(code)); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } QW_UNLOCK(QW_WRITE, &sch->tasksLock); @@ -251,7 +253,7 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { } } else { QW_TASK_ELOG("taosHashPut to ctxHash failed, error:%x", code); - QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -281,7 +283,7 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { int32_t qwKillTaskHandle(SQWTaskCtx *ctx, int32_t rspCode) { int32_t code = 0; - + // Note: free/kill may in RC qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); if (taskHandle && atomic_val_compare_exchange_ptr(&ctx->taskHandle, taskHandle, NULL)) { @@ -363,7 +365,7 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { QW_TASK_ELOG_E("taosHashRemove task from hash failed"); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } QW_TASK_DLOG_E("task status dropped"); @@ -463,6 +465,8 @@ void qwDestroyImpl(void *pMgmt) { int8_t nodeType = mgmt->nodeType; int32_t nodeId = mgmt->nodeId; + int32_t taskCount = 0; + int32_t schStatusCount = 0; qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt); taosTmrStop(mgmt->hbTimer); @@ -472,6 +476,7 @@ void qwDestroyImpl(void *pMgmt) { uint64_t qId, tId; int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); @@ -480,6 +485,7 @@ void qwDestroyImpl(void *pMgmt) { qwFreeTaskCtx(ctx); QW_TASK_DLOG_E("task ctx freed"); pIter = taosHashIterate(mgmt->ctxHash, pIter); + taskCount++; } taosHashCleanup(mgmt->ctxHash); @@ -487,7 +493,9 @@ void qwDestroyImpl(void *pMgmt) { while (pIter) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); + pIter = taosHashIterate(mgmt->schHash, pIter); + schStatusCount++; } taosHashCleanup(mgmt->schHash); @@ -499,7 +507,8 @@ void qwDestroyImpl(void *pMgmt) { qwCloseRef(); - qDebug("qworker destroyed, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt); + qDebug("qworker destroyed, type:%d, id:%d, handle:%p, taskCount:%d, schStatusCount: %d", nodeType, nodeId, mgmt, + taskCount, schStatusCount); } int32_t qwOpenRef(void) { @@ -509,7 +518,7 @@ int32_t qwOpenRef(void) { if (gQwMgmt.qwRef < 0) { taosWUnLockLatch(&gQwMgmt.lock); qError("init qworker ref failed"); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } } taosWUnLockLatch(&gQwMgmt.lock); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9a318df324704291f83b6a69594c06b8823cd053..81f73b1226227d79a1c11cbaa1d162bd462c6bf1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -8,9 +8,9 @@ #include "qwMsg.h" #include "tcommon.h" #include "tdatablock.h" +#include "tglobal.h" #include "tmsg.h" #include "tname.h" -#include "tglobal.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -117,7 +117,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { if (queryStop) { *queryStop = true; } - + return TSDB_CODE_SUCCESS; } @@ -216,7 +216,7 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) if (NULL == hbInfo->rsp.taskStatus) { QW_UNLOCK(QW_READ, &sch->tasksLock); QW_ELOG("taosArrayInit taskStatus failed, num:%d", taskNum); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } void *key = NULL; @@ -275,7 +275,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, QW_ERR_RET(code); } - QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("no more data in sink and query end, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); @@ -320,19 +320,21 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, pOutput->numOfBlocks++; if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { - QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %d", pOutput->numOfBlocks, + QW_TASK_DLOG("task all data fetched and done, fetched blocks %d rows %" PRId64, pOutput->numOfBlocks, pOutput->numOfRows); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC); break; } if (0 == ctx->level) { - QW_TASK_DLOG("task fetched blocks %d rows %d, level %d", pOutput->numOfBlocks, pOutput->numOfRows, ctx->level); + QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 ", level %d", pOutput->numOfBlocks, pOutput->numOfRows, + ctx->level); break; } if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { - QW_TASK_DLOG("task fetched blocks %d rows %d reaches the min rows", pOutput->numOfBlocks, pOutput->numOfRows); + QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, + pOutput->numOfRows); break; } } @@ -463,7 +465,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } default: QW_TASK_ELOG("invalid phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } if (ctx->rspCode) { @@ -508,7 +510,7 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_PHASE_POST_FETCH == phase) { QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } // qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code); @@ -538,7 +540,7 @@ _return: SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); - if (!rsped) { + if (!rsped) { qwSendQueryRsp(QW_FPARAMS(), input->msgType + 1, ctx, code, false); } } @@ -629,7 +631,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); @@ -650,8 +652,8 @@ _return: code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL && QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - void *rsp = NULL; - int32_t dataLen = 0; + void *rsp = NULL; + int32_t dataLen = 0; SOutputData sOutput = {0}; if (qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)) { return TSDB_CODE_SUCCESS; @@ -671,8 +673,8 @@ _return: qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, - tstrerror(code), dataLen); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); } } @@ -689,7 +691,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { do { ctx = NULL; - + QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); @@ -748,7 +750,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } QW_LOCK(QW_WRITE, &ctx->lock); - if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { + if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code || + 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { // Note: query is not running anymore QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -1059,7 +1062,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx.taskHandle = pTaskInfo; @@ -1097,7 +1100,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorker)); atomic_sub_fetch_32(&gQwMgmt.qwNum, 1); - QW_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_RET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->cfg.maxSchedulerNum = QW_DEFAULT_SCHEDULER_NUMBER; @@ -1109,20 +1112,20 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S if (NULL == mgmt->schHash) { taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->timer = taosTmrInit(0, 0, 0, "qworker"); if (NULL == mgmt->timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } mgmt->nodeType = nodeType; @@ -1145,7 +1148,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const S mgmt->hbTimer = taosTmrStart(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, (void *)param, mgmt->timer); if (NULL == mgmt->hbTimer) { qError("start hb timer failed"); - QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + QW_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *qWorkerMgmt = mgmt; @@ -1176,7 +1179,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash)); uint64_t qId, tId; - int32_t eId; + int32_t eId; void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; @@ -1186,7 +1189,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { QW_LOCK(QW_WRITE, &ctx->lock); QW_TASK_DLOG_E("start to force stop task"); - + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_WLOG_E("task already dropping"); QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -1194,7 +1197,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { pIter = taosHashIterate(mgmt->ctxHash, pIter); continue; } - + if (QW_QUERY_RUNNING(ctx)) { qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED); } else if (!QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { @@ -1288,7 +1291,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 if (NULL == sinkHandle || NULL == pTaskInfo) { QW_TASK_ELOG("create task result error, taskHandle:%p, sinkHandle:%p", pTaskInfo, sinkHandle); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + QW_ERR_JRET(TSDB_CODE_APP_ERROR); } ctx->level = plan->level; diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index e26bc59d47e368487370d9f0c260d61b24dc84a7..2023d387773aa294b6949f35365060dc8661c322 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -101,6 +101,11 @@ typedef int32_t (*filter_desc_compare_func)(const void *, const void *); typedef bool (*filter_exec_func)(void *, int32_t, SColumnInfoData *, SColumnDataAgg *, int16_t, int32_t *); typedef int32_t (*filer_get_col_from_name)(void *, int32_t, char *, void **); +typedef struct SFilterDataInfo { + int32_t idx; + void* addr; +} SFilterDataInfo; + typedef struct SFilterRangeCompare { int64_t s; int64_t e; @@ -294,9 +299,9 @@ struct SFilterInfo { #define CHK_OR_OPTR(ctx) ((ctx)->isnull == true && (ctx)->notnull == true) #define CHK_AND_OPTR(ctx) ((ctx)->isnull == true && (((ctx)->notnull == true) || ((ctx)->isrange == true))) -#define FILTER_GET_FLAG(st, f) (st & f) -#define FILTER_SET_FLAG(st, f) st |= (f) -#define FILTER_CLR_FLAG(st, f) st &= (~f) +#define FILTER_GET_FLAG(st, f) ((st) & (f)) +#define FILTER_SET_FLAG(st, f) (st) |= (f) +#define FILTER_CLR_FLAG(st, f) (st) &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) #define FLT_PACKAGE_UNIT_HASH_KEY(v, op1, op2, lidx, ridx, ridx2) \ diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 45931c209c20017027c50db214d1368e023e9469..59e39e3f6f37b4a29c98f114d8dc662dba45ddad 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -833,7 +833,7 @@ int32_t filterGetRangeRes(void *h, SFilterRange *ra) { if (num == 0) { qError("no range result"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -963,16 +963,17 @@ int32_t filterGetFiledByDesc(SFilterFields *fields, int32_t type, void *v) { return -1; } -int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen) { +int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t dataLen, bool *sameBuf) { if (type == FLD_TYPE_VALUE) { if (info->pctx.valHash == false) { qError("value hash is empty"); return -1; } - void *hv = taosHashGet(info->pctx.valHash, v, dataLen); - if (hv) { - return *(int32_t *)hv; + SFilterDataInfo *dInfo = taosHashGet(info->pctx.valHash, v, dataLen); + if (dInfo) { + *sameBuf = (dInfo->addr == v); + return dInfo->idx; } } @@ -982,9 +983,10 @@ int32_t filterGetFiledByData(SFilterInfo *info, int32_t type, void *v, int32_t d // In the params, we should use void *data instead of void **data, there is no need to use taosMemoryFreeClear(*data) to // set *data = 0 Besides, fields data value is a pointer, so dataLen should be POINTER_BYTES for better. int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, SFilterFieldId *fid, int32_t dataLen, - bool freeIfExists) { + bool freeIfExists, int16_t *srcFlag) { int32_t idx = -1; uint32_t *num; + bool sameBuf = false; num = &info->fields[type].num; @@ -992,7 +994,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, if (type == FLD_TYPE_COLUMN) { idx = filterGetFiledByDesc(&info->fields[type], type, desc); } else if (data && (*data) && dataLen > 0 && FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { - idx = filterGetFiledByData(info, type, *data, dataLen); + idx = filterGetFiledByData(info, type, *data, dataLen, &sameBuf); } } @@ -1020,11 +1022,17 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); } - taosHashPut(info->pctx.valHash, *data, dataLen, &idx, sizeof(idx)); + SFilterDataInfo dInfo = {idx, *data}; + taosHashPut(info->pctx.valHash, *data, dataLen, &dInfo, sizeof(dInfo)); + if (srcFlag) { + FILTER_SET_FLAG(*srcFlag, FLD_DATA_NO_FREE); + } } - } else { - if (data && freeIfExists) { + } else if (type != FLD_TYPE_COLUMN && data) { + if (freeIfExists) { taosMemoryFreeClear(*data); + } else if (sameBuf) { + FILTER_SET_FLAG(*srcFlag, FLD_DATA_NO_FREE); } } @@ -1035,7 +1043,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, } static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilterField *field, SFilterFieldId *fid) { - filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false); + filterAddField(info, field->desc, &field->data, FILTER_GET_TYPE(field->flag), fid, 0, false, NULL); FILTER_SET_FLAG(field->flag, FLD_DATA_NO_FREE); @@ -1045,12 +1053,12 @@ static FORCE_INLINE int32_t filterAddColFieldFromField(SFilterInfo *info, SFilte int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *fid) { if (node == NULL) { fltError("empty node"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } int32_t type; @@ -1064,7 +1072,7 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f v = node; } - filterAddField(info, v, NULL, type, fid, 0, true); + filterAddField(info, v, NULL, type, fid, 0, true, NULL); return TSDB_CODE_SUCCESS; } @@ -1117,7 +1125,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, int32_t paramNum = scalarGetOperatorParamNum(optr); if (1 != paramNum) { fltError("invalid right field in unit, operator:%s, rightType:%d", operatorTypeStr(optr), u->right.type); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } } @@ -1195,15 +1203,15 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode *tree, SArray *group) { len = tDataTypes[type].bytes; - filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, (void **)&out.columnData->pData, FLD_TYPE_VALUE, &right, len, true, NULL); out.columnData->pData = NULL; } else { void *data = taosMemoryCalloc(1, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); // reserved space for simple_copy if (NULL == data) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(data, nodesGetValueFromNode(valueNode), tDataTypes[type].bytes); - filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, (void **)&data, FLD_TYPE_VALUE, &right, len, true, NULL); } filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); @@ -1234,28 +1242,26 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u uint8_t type = FILTER_UNIT_DATA_TYPE(u); uint16_t flag = 0; - filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false); + filterAddField(dst, FILTER_UNIT_COL_DESC(src, u), NULL, FLD_TYPE_COLUMN, &left, 0, false, NULL); SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); if (u->right.type == FLD_TYPE_VALUE) { void *data = FILTER_UNIT_VAL_DATA(src, u); + SFilterField *rField = FILTER_UNIT_RIGHT_FIELD(src, u); + if (IS_VAR_DATA_TYPE(type)) { if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, - false); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. + false, &rField->flag); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, varDataTLen(data), false, &rField->flag); } } else { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, false, &rField->flag); } - - flag = FLD_DATA_NO_FREE; - t = FILTER_UNIT_RIGHT_FIELD(src, u); - FILTER_SET_FLAG(t->flag, flag); } else { pright = NULL; } @@ -1313,17 +1319,17 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (func(&ra->s, &ra->e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } else { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &ra->e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true, NULL); filterAddUnitImpl( dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, @@ -1337,7 +1343,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1346,7 +1352,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(ra->eflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &ra->e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1393,16 +1399,16 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (func(&r->ra.s, &r->ra.e) == 0) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, OP_TYPE_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); } else { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); void *data2 = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data2, &r->ra.e); - filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true, NULL); filterAddUnitImpl( dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, @@ -1421,7 +1427,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.s); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1430,7 +1436,7 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan if (!FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_NULL)) { void *data = taosMemoryMalloc(sizeof(int64_t)); SIMPLE_COPY_VALUES(data, &r->ra.e); - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true); + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, tDataTypes[type].bytes, true, NULL); filterAddUnit(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &left, &right, &uidx); filterAddUnitToGroup(g, uidx); @@ -1509,7 +1515,7 @@ EDealRes fltTreeToGroup(SNode *pNode, void *pContext) { return DEAL_RES_IGNORE_CHILD; } - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; fltError("invalid condition type, type:%d", node->condType); @@ -1940,7 +1946,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { FLT_ERR_RET(scalarGenerateSetFromList((void **)&fi->data, fi->desc, type)); if (fi->data == NULL) { fltError("failed to convert in param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } FILTER_SET_FLAG(fi->flag, FLD_DATA_IS_HASH); @@ -2008,7 +2014,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0) { qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -3715,12 +3721,12 @@ int32_t fltAddValueNodeToConverList(SFltTreeStat *stat, SValueNode *pNode) { if (NULL == stat->nodeList) { stat->nodeList = taosArrayInit(10, POINTER_BYTES); if (NULL == stat->nodeList) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(stat->nodeList, &pNode)) { - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; @@ -3841,7 +3847,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3902,7 +3908,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { if (OP_TYPE_IN == node->opType && QUERY_NODE_NODE_LIST != nodeType(node->pRight)) { fltError("invalid IN operator node, rightType:%d", nodeType(node->pRight)); - stat->code = TSDB_CODE_QRY_APP_ERROR; + stat->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -3988,7 +3994,7 @@ int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, id); @@ -4018,14 +4024,14 @@ int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pInfo, uint32_t options) int32_t code = 0; if (pNode == NULL || pInfo == NULL) { fltError("invalid param"); - FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + FLT_ERR_RET(TSDB_CODE_APP_ERROR); } if (*pInfo == NULL) { *pInfo = taosMemoryCalloc(1, sizeof(SFilterInfo)); if (NULL == *pInfo) { fltError("taosMemoryCalloc %d failed", (int32_t)sizeof(SFilterInfo)); - FLT_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + FLT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 55f33b7a3e50acf067935288bf285d6bed2f1144..e3436c83c1f50739d83295ace1deab49ccfca4c2 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -83,7 +83,7 @@ int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockL SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); if (NULL == pLeft) { sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pLeft->numOfRows = pb->info.rows; @@ -104,7 +104,7 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); if (NULL == pObj) { sclError("taosHashInit failed, size:%d", 256); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); @@ -162,13 +162,10 @@ int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type) { if (taosHashPut(pObj, buf, (size_t)len, NULL, 0)) { sclError("taosHashPut to set failed"); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - colDataDestroy(out.columnData); - taosMemoryFreeClear(out.columnData); - out.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); - + colInfoDataCleanup(out.columnData, out.numOfRows); cell = cell->pNext; } @@ -224,7 +221,7 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { *res = taosMemoryMalloc(pNode->node.resType.bytes); if (NULL == (*res)) { sclError("malloc %d failed", pNode->node.resType.bytes); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); @@ -362,7 +359,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t taosHashCleanup(param->pHashFilter); param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; } param->colAlloced = false; break; @@ -370,7 +367,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_COLUMN: { if (NULL == ctx->pBlockList) { sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } SColumnNode *ref = (SColumnNode *)node; @@ -417,7 +414,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_RET(TSDB_CODE_APP_ERROR); } *param = *res; param->colAlloced = false; @@ -459,7 +456,7 @@ int32_t sclInitParamList(SScalarParam **pParams, SNodeList *pParamList, SScalarC SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pParamList) { @@ -548,7 +545,7 @@ int32_t sclInitOperatorParams(SScalarParam **pParams, SOperatorNode *node, SScal SScalarParam *paramList = taosMemoryCalloc(paramNum, sizeof(SScalarParam)); if (NULL == paramList) { sclError("calloc %d failed", (int32_t)(paramNum * sizeof(SScalarParam))); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } sclSetOperatorValueType(node, ctx); @@ -987,7 +984,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -999,7 +996,7 @@ EDealRes sclRewriteNullInOptr(SNode **pNode, SScalarCtx *ctx, EOperatorType opTy SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1126,7 +1123,7 @@ EDealRes sclRewriteFunction(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1178,7 +1175,7 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1218,7 +1215,7 @@ EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1270,7 +1267,7 @@ EDealRes sclRewriteCaseWhen(SNode **pNode, SScalarCtx *ctx) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1329,7 +1326,7 @@ EDealRes sclWalkFunction(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1346,7 +1343,7 @@ EDealRes sclWalkLogic(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1364,7 +1361,7 @@ EDealRes sclWalkOperator(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1412,7 +1409,7 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); - ctx->code = TSDB_CODE_QRY_APP_ERROR; + ctx->code = TSDB_CODE_APP_ERROR; return DEAL_RES_ERROR; } @@ -1434,7 +1431,7 @@ EDealRes sclWalkCaseWhen(SNode *pNode, SScalarCtx *ctx) { } if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { - ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -1485,7 +1482,7 @@ int32_t sclCalcConstants(SNode *pNode, bool dual, SNode **pRes) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesRewriteExprPostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); @@ -1603,7 +1600,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); - SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } nodesWalkExprPostOrder(pNode, sclCalcWalker, (void *)&ctx); @@ -1613,7 +1610,7 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); if (NULL == res) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCL_ERR_JRET(TSDB_CODE_APP_ERROR); } if (1 == res->numOfRows) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d261d572f0fb9997f6b72640cfb83aac9f8390e8..1de8a35308b4cf35982ee9e028b67ba1f7db9960 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1174,12 +1174,35 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } /** Time functions **/ +static int64_t offsetFromTz(char *timezone, int64_t factor) { + char *minStr = &timezone[3]; + int64_t minutes = taosStr2Int64(minStr, NULL, 10); + memset(minStr, 0, strlen(minStr)); + int64_t hours = taosStr2Int64(timezone, NULL, 10); + int64_t seconds = hours * 3600 + minutes * 60; + + return seconds * factor; + +} + int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(&pInput[0]); int64_t timeUnit, timePrec, timeVal = 0; + bool ignoreTz = true; + char timezone[20] = {0}; + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + + int32_t timePrecIdx = 2, timeZoneIdx = 3; + if (inputNum == 5) { + timePrecIdx += 1; + timeZoneIdx += 1; + GET_TYPED_DATA(ignoreTz, bool, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + } + + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); + memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t unit = timeUnit * 1000 / factor; @@ -1294,13 +1317,29 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } case 86400000: { /* 1d */ if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000 / 86400 * 86400 * 1000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (86400L * 1000); + } else { + timeVal = timeVal / 1000 / 86400 * 86400 * 1000; + } } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (86400L * 1000000); + } else { + timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; + } } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (86400L * 1000000000); + } else { + timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; + } } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor / factor / 86400 * 86400 * factor; + if (ignoreTz) { + timeVal = (timeVal - (timeVal + offsetFromTz(timezone, 1)) % (86400L)) * factor; + } else { + timeVal = timeVal * factor / factor / 86400 * 86400 * factor; + } } else { colDataAppendNULL(pOutput->columnData, i); continue; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c4ff5b2b01cc13a888319cf2a25afd14f975ddfa..a1995bdf500bedaa906c488fa5769146b41a8741 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -395,7 +395,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { func = varToTimestamp; } else { sclError("invalid convert outType:%d", pCtx->outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } pCtx->pOut->numOfRows = pCtx->pIn->numOfRows; @@ -440,7 +440,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) { if (len < 0) { sclError("castConvert taosUcs4ToMbs error 1"); taosMemoryFreeClear(tmp); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } tmp[len] = 0; @@ -642,7 +642,7 @@ int32_t vectorConvertToVarData(SSclVectorConvCtx *pCtx) { } } else { sclError("not supported input type:%d", pCtx->inType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -859,7 +859,7 @@ int32_t vectorConvertSingleColImpl(const SScalarParam *pIn, SScalarParam *pOut, } default: sclError("invalid convert output type:%d", cCtx.outType); - return TSDB_CODE_QRY_APP_ERROR; + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -1543,9 +1543,44 @@ void vectorBitOr(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, int32_t startIndex, int32_t numOfRows, int32_t step, __compar_fn_t fp, int32_t optr) { int32_t num = 0; - bool * pRes = (bool *)pOut->columnData->pData; + bool *pRes = (bool *)pOut->columnData->pData; + + if (IS_MATHABLE_TYPE(GET_PARAM_TYPE(pLeft)) && IS_MATHABLE_TYPE(GET_PARAM_TYPE(pRight))) { + if (!(pLeft->columnData->hasNull || pRight->columnData->hasNull)) { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } else { + for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { + int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; + int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; - if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { + if (colDataIsNull_f(pLeft->columnData->nullbitmap, leftIndex) || + colDataIsNull_f(pRight->columnData->nullbitmap, rightIndex)) { + pRes[i] = false; + continue; + } + + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); + + pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); + if (pRes[i]) { + ++num; + } + } + } + } else { + // if (GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_JSON || GET_PARAM_TYPE(pRight) == TSDB_DATA_TYPE_JSON) { for (int32_t i = startIndex; i < numOfRows && i >= startIndex; i += step) { int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; @@ -1556,8 +1591,8 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa continue; } - char * pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char * pRightData = colDataGetData(pRight->columnData, rightIndex); + char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); + char *pRightData = colDataGetData(pRight->columnData, rightIndex); int64_t leftOut = 0; int64_t rightOut = 0; bool freeLeft = false; @@ -1592,25 +1627,6 @@ int32_t doVectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarPa taosMemoryFreeClear(pRightData); } } - } else { - for (int32_t i = startIndex; i < numOfRows && i >= 0; i += step) { - int32_t leftIndex = (i >= pLeft->numOfRows) ? 0 : i; - int32_t rightIndex = (i >= pRight->numOfRows) ? 0 : i; - - if (colDataIsNull_s(pLeft->columnData, leftIndex) || - colDataIsNull_s(pRight->columnData, rightIndex)) { - pRes[i] = false; - continue; - } - - char *pLeftData = colDataGetData(pLeft->columnData, leftIndex); - char *pRightData = colDataGetData(pRight->columnData, rightIndex); - - pRes[i] = filterDoCompare(fp, optr, pLeftData, pRightData); - if (pRes[i]) { - ++num; - } - } } return num; @@ -1766,7 +1782,7 @@ void vectorIsTrue(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *pOut, if (colDataIsNull_s(pOut->columnData, i)) { int8_t v = 0; colDataAppendInt8(pOut->columnData, i, &v); - colDataSetNotNull_f(pOut->columnData->nullbitmap, i); + colDataClearNull_f(pOut->columnData->nullbitmap, i); } } pOut->columnData->hasNull = false; diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index dfc48e7d9fba21ed66f60089546762162050f390..48df7e36a30737324342bb06bdbbfced8b73a5cd 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -293,11 +293,12 @@ typedef struct SSchJob { void *chkKillParam; SSchTask *fetchTask; int32_t errCode; + int32_t redirectCode; SRWLatch resLock; SExecResult execRes; void *fetchRes; // TODO free it or not bool fetched; - int32_t resNumOfRows; + int64_t resNumOfRows; // from int32_t to int64_t SSchResInfo userRes; char *sql; SQueryProfileSummary summary; @@ -331,6 +332,9 @@ extern SSchedulerMgmt schMgmt; ((_job)->attr.localExec && SCH_IS_QUERY_JOB(_job) && (!SCH_IS_INSERT_JOB(_job)) && \ (!SCH_IS_DATA_BIND_QRY_TASK(_task))) +#define SCH_UPDATE_REDICT_CODE(job, _code) atomic_val_compare_exchange_32(&((job)->redirectCode), 0, _code) +#define SCH_GET_REDICT_CODE(job, _code) (((!NO_RET_REDIRECT_ERROR(_code)) || (job)->redirectCode == 0) ? (_code) : (job)->redirectCode) + #define SCH_SET_TASK_STATUS(task, st) atomic_store_8(&(task)->status, st) #define SCH_GET_TASK_STATUS(task) atomic_load_8(&(task)->status) #define SCH_GET_TASK_STATUS_STR(task) jobTaskStatusStr(SCH_GET_TASK_STATUS(task)) @@ -373,7 +377,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) \ - (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) + (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)) || (_task)->redirectCtx.inRedirect)) #define SCH_REDIRECT_MSGTYPE(_msgType) \ ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || \ (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 8980d08e891d58f4bbc7e1e88d0d3ea5d1c067ee..5e4fe4b8a1f2795cfbd4dff838304fe0e36343ff 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -63,7 +63,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { taosHashInit(pJob->taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == pJob->flowCtrl) { SCH_JOB_ELOG("taosHashInit %d flowCtrl failed", pJob->taskNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_JOB_NEED_FLOW_CTRL(pJob); @@ -122,7 +122,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { } SCH_TASK_ELOG("taosHashPut flowCtrl failed, size:%d", (int32_t)sizeof(nctrl)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, @@ -156,13 +156,13 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { ctrl->taskList = taosArrayInit(pLevel->taskNum, POINTER_BYTES); if (NULL == ctrl->taskList) { SCH_TASK_ELOG("taosArrayInit taskList failed, size:%d", (int32_t)pLevel->taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(ctrl->taskList, &pTask)) { SCH_TASK_ELOG("taosArrayPush to taskList failed, size:%d", (int32_t)taosArrayGetSize(ctrl->taskList)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } *enough = false; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 0eb29a36670e7bcf2463e5f09fc90697bfc2ff14..d422f0e88f104749dc92804b46d38aca6f89b40b 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -89,34 +89,34 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { switch (oriStatus) { case JOB_TASK_STATUS_NULL: if (newStatus != JOB_TASK_STATUS_INIT) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_INIT: if (newStatus != JOB_TASK_STATUS_EXEC && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_EXEC: if (newStatus != JOB_TASK_STATUS_PART_SUCC && newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; case JOB_TASK_STATUS_SUCC: case JOB_TASK_STATUS_FAIL: if (newStatus != JOB_TASK_STATUS_DROP) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } break; @@ -125,7 +125,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { default: SCH_JOB_ELOG("invalid job status:%s", jobTaskStatusStr(oriStatus)); - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_JRET(TSDB_CODE_APP_ERROR); } if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { @@ -168,7 +168,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->children = taosArrayInit(childNum, POINTER_BYTES); if (NULL == pTask->children) { SCH_TASK_ELOG("taosArrayInit %d children failed", childNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -182,7 +182,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->children, childTask)) { SCH_TASK_ELOG("taosArrayPush childTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("children info, the %d child TID 0x%" PRIx64, n, (*childTask)->taskId); @@ -197,7 +197,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { pTask->parents = taosArrayInit(parentNum, POINTER_BYTES); if (NULL == pTask->parents) { SCH_TASK_ELOG("taosArrayInit %d parents failed", parentNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } else { if (0 != pLevel->level) { @@ -216,7 +216,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { if (NULL == taosArrayPush(pTask->parents, parentTask)) { SCH_TASK_ELOG("taosArrayPush parentTask failed, level:%d, taskIdx:%d, childIdx:%d", i, m, n); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("parents info, the %d parent TID 0x%" PRIx64, n, (*parentTask)->taskId); @@ -278,13 +278,13 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); if (NULL == pJob->levels) { SCH_JOB_ELOG("taosArrayInit %d failed", levelNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->levelNum = levelNum; @@ -300,7 +300,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t i = 0; i < levelNum; ++i) { if (NULL == taosArrayPush(pJob->levels, &level)) { SCH_JOB_ELOG("taosArrayPush level failed, level:%d", i); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pLevel = taosArrayGet(pJob->levels, i); @@ -323,7 +323,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); if (NULL == pLevel->subTasks) { SCH_JOB_ELOG("taosArrayInit %d failed", taskNum); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } for (int32_t n = 0; n < taskNum; ++n) { @@ -335,7 +335,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchTask *pTask = taosArrayPush(pLevel->subTasks, &task); if (NULL == pTask) { SCH_TASK_ELOG("taosArrayPush task to level failed, level:%d, taskIdx:%d", pLevel->level, n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schInitTask(pJob, pTask, plan, pLevel)); @@ -344,12 +344,12 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (0 != taosHashPut(pJob->taskList, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES)) { SCH_TASK_ELOG("taosHashPut to taskList failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } ++pJob->taskNum; @@ -412,7 +412,7 @@ int32_t schDumpJobFetchRes(SSchJob *pJob, void **pData) { SCH_JOB_DLOG("empty res and set query complete, code:%x", code); } - SCH_JOB_DLOG("fetch done, totalRows:%d", pJob->resNumOfRows); + SCH_JOB_DLOG("fetch done, totalRows:%" PRId64, pJob->resNumOfRows); _return: @@ -481,6 +481,10 @@ _return: } int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { + return TSDB_CODE_SCH_IGNORE_ERROR; + } + schUpdateJobErrCode(pJob, errCode); int32_t code = atomic_load_32(&pJob->errCode); @@ -526,9 +530,9 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { void schProcessOnDataFetched(SSchJob *pJob) { schPostJobRes(pJob, SCH_OP_FETCH); } int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp) { - SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); + SCH_TASK_DLOG("got explain rsp, rows:%" PRId64 ", complete:%d", htobe64(pRsp->numOfRows), pRsp->completed); - atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); + atomic_store_64(&pJob->resNumOfRows, htobe64(pRsp->numOfRows)); atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -702,7 +706,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pJob->attr.explainMode = pReq->pDag->explainInfo.mode; @@ -728,7 +732,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schValidateAndBuildJob(pReq->pDag, pJob)); @@ -741,7 +745,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { HASH_ENTRY_LOCK); if (NULL == pJob->execTasks) { SCH_JOB_ELOG("taosHashInit %d execTasks failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } tsem_init(&pJob->rspSem, 0, 0); @@ -865,8 +869,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -879,8 +883,8 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (SCH_OP_NULL != atomic_val_compare_exchange_32(&pJob->opStatus.op, SCH_OP_NULL, type)) { SCH_JOB_ELOG("job already in %s operation", schGetOpStr(pJob->opStatus.op)); SCH_UNLOCK(SCH_WRITE, &pJob->opStatus.lock); - schDirectPostJobRes(pReq, TSDB_CODE_TSC_APP_ERROR); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + schDirectPostJobRes(pReq, TSDB_CODE_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_JOB_DLOG("job start %s operation", schGetOpStr(pJob->opStatus.op)); @@ -894,7 +898,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq if (!SCH_JOB_NEED_FETCH(pJob)) { SCH_JOB_ELOG("no need to fetch data, status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (status != JOB_TASK_STATUS_PART_SUCC) { @@ -911,7 +915,7 @@ int32_t schProcessOnOpBegin(SSchJob *pJob, SCH_OP_TYPE type, SSchedulerReq *pReq return TSDB_CODE_SUCCESS; default: SCH_JOB_ELOG("unknown operation type %d", type); - SCH_ERR_RET(TSDB_CODE_TSC_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } if (schJobNeedToStop(pJob, &status)) { diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 4915bba0853bc5afe10367726ce4d990d04a4902..f3ef8c4afcf9fe0eb2ec393425ae05b95d515913 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -108,13 +108,13 @@ int32_t schProcessFetchRsp(SSchJob *pJob, SSchTask *pTask, char *msg, int32_t rs } atomic_store_ptr(&pJob->fetchRes, rsp); - atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); + atomic_add_fetch_64(&pJob->resNumOfRows, htobe64(rsp->numOfRows)); if (rsp->completed) { SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); } - SCH_TASK_DLOG("got fetch rsp, rows:%d, complete:%d", htonl(rsp->numOfRows), rsp->completed); + SCH_TASK_DLOG("got fetch rsp, rows:%" PRId64 ", complete:%d", htobe64(rsp->numOfRows), rsp->completed); msg = NULL; schProcessOnDataFetched(pJob); @@ -156,6 +156,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_RET(schHandleRedirect(pJob, pTask, (SDataBuf *)pMsg, rspCode)); } + pTask->redirectCtx.inRedirect = false; + switch (msgType) { case TDMT_VND_COMMIT_RSP: { SCH_ERR_JRET(rspCode); @@ -262,13 +264,15 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); code = tDecodeSSubmitRsp2(&coder, rsp); + tDecoderClear(&coder); if (code) { SCH_TASK_ELOG("tDecodeSSubmitRsp2 failed, code:%d", code); tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); SCH_ERR_JRET(code); } - atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); @@ -281,11 +285,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (sum->aCreateTbRsp) { taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); taosArrayDestroy(rsp->aCreateTbRsp); - taosMemoryFree(rsp); } else { TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); - taosMemoryFree(rsp); } + taosMemoryFree(rsp); } else { pJob->execRes.res = rsp; pJob->execRes.msgType = TDMT_VND_SUBMIT; @@ -301,6 +304,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); } } @@ -320,7 +324,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa tDecodeSVDeleteRsp(&coder, &rsp); tDecoderClear(&coder); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); SCH_TASK_DLOG("delete succeed, affectedRows:%" PRId64, rsp.affectedRows); } @@ -347,7 +351,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(schSaveJobExecRes(pJob, &rsp)); - atomic_add_fetch_32(&pJob->resNumOfRows, rsp.affectedRows); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp.affectedRows); taosMemoryFreeClear(msg); @@ -374,7 +378,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SExplainRsp rsp = {0}; if (tDeserializeSExplainRsp(msg, msgSize, &rsp)) { tFreeSExplainRsp(&rsp); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schProcessExplainRsp(pJob, pTask, &rsp)); @@ -509,7 +513,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchTaskCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->queryId = pJob->queryId; @@ -526,7 +530,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -546,7 +550,7 @@ int32_t schMakeCallbackParam(SSchJob *pJob, SSchTask *pTask, int32_t msgType, bo SSchTaskCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchTaskCallbackParam)); if (NULL == param) { qError("calloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->pTrans = trans->pTrans; @@ -561,7 +565,7 @@ int32_t schGenerateCallBackInfo(SSchJob *pJob, SSchTask *pTask, void *msg, uint3 SMsgSendInfo *msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == msgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } msgSendInfo->paramFreeFp = taosMemoryFree; @@ -624,7 +628,7 @@ int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) { break; default: qError("unknown msg type for callback, msgType:%d", msgType); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } return TSDB_CODE_SUCCESS; @@ -635,7 +639,7 @@ int32_t schMakeHbCallbackParam(SSchJob *pJob, SSchTask *pTask, void **pParam) { SSchHbCallbackParam *param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->head.isHbParam = true; @@ -665,7 +669,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { pDst->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pDst->args) { qError("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SRpcCtxVal dst = {0}; @@ -682,7 +686,7 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { if (taosHashPut(pDst->args, msgType, sizeof(*msgType), &dst, sizeof(dst))) { qError("taosHashPut msg %d to rpcCtx failed", *msgType); (*pSrc->freeFunc)(dst.val); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } pIter = taosHashIterate(pSrc->args, pIter); @@ -709,19 +713,19 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SMsgSendInfo)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } param = taosMemoryCalloc(1, sizeof(SSchHbCallbackParam)); if (NULL == param) { SCH_TASK_ELOG("calloc %d failed", (int32_t)sizeof(SSchHbCallbackParam)); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int32_t msgType = TDMT_SCH_QUERY_HEARTBEAT_RSP; @@ -738,7 +742,7 @@ int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, true)); @@ -783,7 +787,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { pCtx->args = taosHashInit(1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); if (NULL == pCtx->args) { SCH_TASK_ELOG("taosHashInit %d RpcCtx failed", 1); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SSchTrans trans = {.pTrans = pJob->conn.pTrans, .pHandle = SCH_GET_TASK_HANDLE(pTask)}; @@ -793,7 +797,7 @@ int32_t schMakeQueryRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { SRpcCtxVal ctxVal = {.val = pExplainMsgSendInfo, .clone = schCloneSMsgSendInfo}; if (taosHashPut(pCtx->args, &msgType, sizeof(msgType), &ctxVal, sizeof(ctxVal))) { SCH_TASK_ELOG("taosHashPut msg %d to rpcCtx failed", msgType); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_ERR_JRET(schMakeBrokenLinkVal(pJob, pTask, &pCtx->brokenVal, false)); @@ -818,7 +822,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchHbCallbackParam *dst = taosMemoryMalloc(sizeof(SSchHbCallbackParam)); if (NULL == dst) { qError("malloc SSchHbCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -830,7 +834,7 @@ int32_t schCloneCallbackParam(SSchCallbackParamHeader *pSrc, SSchCallbackParamHe SSchTaskCallbackParam *dst = taosMemoryMalloc(sizeof(SSchTaskCallbackParam)); if (NULL == dst) { qError("malloc SSchTaskCallbackParam failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(dst, pSrc, sizeof(*dst)); @@ -845,7 +849,7 @@ int32_t schCloneSMsgSendInfo(void *src, void **dst) { SMsgSendInfo *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == pDst) { qError("malloc SMsgSendInfo for rpcCtx failed, len:%d", (int32_t)sizeof(*pSrc)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pDst, pSrc, sizeof(*pSrc)); @@ -962,17 +966,17 @@ int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray *taskAction) { int32_t msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } void *msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { qError("calloc hb req %d failed", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { qError("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } int64_t transporterId = 0; @@ -1019,7 +1023,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(msg, pTask->msg, msgSize); @@ -1040,7 +1044,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } tSerializeSVDeleteReq(msg, msgSize, &req); @@ -1070,19 +1074,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSubQueryMsg(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSubQueryMsg(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; @@ -1101,18 +1105,18 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSResFetchReq(NULL, 0, &req); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1129,19 +1133,19 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSTaskDropReq(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq get size, msgSize:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSTaskDropReq(msg, msgSize, &qMsg) < 0) { SCH_TASK_ELOG("tSerializeSTaskDropReq failed, msgSize:%d", msgSize); taosMemoryFree(msg); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } break; } @@ -1157,16 +1161,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, msgSize = tSerializeSSchedulerHbReq(NULL, 0, &req); if (msgSize < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_JOB_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (tSerializeSSchedulerHbReq(msg, msgSize, &req) < 0) { SCH_JOB_ELOG("tSerializeSSchedulerHbReq hbReq failed, size:%d", msgSize); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } persistHandle = true; diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 66d58c95d1c3a36938fb0b5b1fa420c558a33e4a..4c16a81a05003166978dd9ee0f96b2c876f6fefd 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -64,7 +64,7 @@ _return: int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { SSchJob* pJob = schAcquireJob(jobId); if (NULL == pJob) { - qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); + qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index c8b8db367fd4875b979e3e8de7b103454b07cc55..8e60222ca6d4d53b189cffaf05fbae8c7f5dcdde 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -73,7 +73,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * taosHashInit(pTask->maxExecTimes, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); @@ -112,7 +112,7 @@ int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad if (taosHashPut(pTask->execNodes, &execId, sizeof(execId), &nodeInfo, sizeof(nodeInfo))) { SCH_TASK_ELOG("taosHashPut nodeInfo to execNodes failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task execNode added, execId:%d, handle:%p", execId, nodeInfo.handle); @@ -340,7 +340,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) { +int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet, int32_t rspCode) { SSchRedirectCtx *pCtx = &pTask->redirectCtx; if (!pCtx->inRedirect) { pCtx->inRedirect = true; @@ -362,17 +362,12 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) } pCtx->totalTimes++; + pCtx->roundTimes++; if (SCH_IS_DATA_BIND_TASK(pTask) && pEpSet) { pCtx->roundTotal = pEpSet->numOfEps; - pCtx->roundTimes = 0; - - pTask->delayExecMs = 0; - - goto _return; } - pCtx->roundTimes++; if (pCtx->roundTimes >= pCtx->roundTotal) { int64_t nowTs = taosGetTimestampMs(); @@ -380,7 +375,7 @@ int32_t schChkUpdateRedirectCtx(SSchJob *pJob, SSchTask *pTask, SEpSet *pEpSet) if (lastTime > tsMaxRetryWaitTime) { SCH_TASK_DLOG("task no more redirect retry since timeout, now:%" PRId64 ", start:%" PRId64 ", max:%d, total:%d", nowTs, pCtx->startTs, tsMaxRetryWaitTime, pCtx->totalTimes); - SCH_ERR_RET(TSDB_CODE_TIMEOUT_ERROR); + SCH_ERR_RET(SCH_GET_REDICT_CODE(pJob, rspCode)); } pCtx->periodMs *= tsRedirectFactor; @@ -415,7 +410,11 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->retryTimes = 0; } - SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL)); + if (!NO_RET_REDIRECT_ERROR(rspCode)) { + SCH_UPDATE_REDICT_CODE(pJob, rspCode); + } + + SCH_ERR_JRET(schChkUpdateRedirectCtx(pJob, pTask, pData ? pData->pEpSet : NULL, rspCode)); pTask->waitRetry = true; @@ -530,7 +529,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("task added to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); @@ -555,7 +554,7 @@ int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to succTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -582,7 +581,7 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to failTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -607,7 +606,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { } SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } *moved = true; @@ -712,7 +711,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { if (NULL == taosArrayPush(pTask->candidateAddrs, naddr)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, addNum:%d, errno:%d", addNum, errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, @@ -740,13 +739,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (pTask->plan->execNode.epSet.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } SCH_TASK_DLOG("use execNode in plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); @@ -756,7 +755,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); @@ -1173,7 +1172,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { SSchTimerParam *param = taosMemoryMalloc(sizeof(SSchTimerParam)); if (NULL == param) { SCH_TASK_ELOG("taosMemoryMalloc %d failed", (int)sizeof(SSchTimerParam)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } param->rId = pJob->refId; @@ -1184,7 +1183,7 @@ int32_t schDelayLaunchTask(SSchJob *pJob, SSchTask *pTask) { pTask->delayTimer = taosTmrStart(schHandleTimerEvent, pTask->delayExecMs, (void *)param, schMgmt.timer); if (NULL == pTask->delayTimer) { SCH_TASK_ELOG("start delay timer failed, handle:%p", schMgmt.timer); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index a5a15cd1c605deaee4b43e0f3d6a0c6f1953f84b..0582184730575717f030b6571ec43eac900c089c 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -235,7 +235,7 @@ int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans) { if (NULL == hb) { SCH_UNLOCK(SCH_READ, &schMgmt.hbLock); qError("taosHashGet hb connection failed, nodeId:%d, fqdn:%s, port:%d", epId->nodeId, epId->ep.fqdn, epId->ep.port); - SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + SCH_ERR_RET(TSDB_CODE_APP_ERROR); } SCH_LOCK(SCH_WRITE, &hb->lock); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 91c3b5d7a142374e0dd4d17cfcc407325ea599b7..7cd5e957b6b8d47871274f80632a65ec11485753 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -39,19 +39,19 @@ int32_t schedulerInit() { schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { qError("init schduler jobRef failed, num:%u", schMgmt.cfg.maxJobNum); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.hbConnections = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == schMgmt.hbConnections) { qError("taosHashInit hb connections failed"); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } schMgmt.timer = taosTmrInit(0, 0, 0, "scheduler"); if (NULL == schMgmt.timer) { qError("init timer failed, error:%s", tstrerror(terrno)); - SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } if (taosGetSystemUUID((char *)&schMgmt.sId, sizeof(schMgmt.sId))) { @@ -157,7 +157,7 @@ void schedulerFreeJob(int64_t *jobId, int32_t errCode) { SSchJob *pJob = schAcquireJob(*jobId); if (NULL == pJob) { - qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 79549675a356ddc716dc6a5a87e26bb3bdcd005c..5b542dd54b1273dc89cec261f3f0b04bd9d35b14 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -56,7 +56,7 @@ void streamSchedByTimer(void* param, void* tmrId) { } if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); + SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM, 0); if (trigger == NULL) return; trigger->type = STREAM_INPUT__GET_RES; trigger->pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); @@ -112,7 +112,7 @@ int32_t streamSchedExec(SStreamTask* pTask) { } int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); int8_t status; // enqueue @@ -150,7 +150,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR } int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); int8_t status = TASK_INPUT_STATUS__NORMAL; // enqueue diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 2fea5b9eca4fb256a8aa1a5de2ce8570d414d121..3f2feb9d286a1cad87c45fe24bc0b5132b873a7f 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -66,12 +66,13 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { + SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; @@ -80,47 +81,49 @@ FAIL: return NULL; } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM); +SStreamMergedSubmit2* streamMergedSubmitNew() { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) return NULL; - pMerged->reqs = taosArrayInit(0, sizeof(void*)); + pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->reqs == NULL) goto FAIL; + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; FAIL: - if (pMerged->reqs) taosArrayDestroy(pMerged->reqs); + if (pMerged->submits) taosArrayDestroy(pMerged->submits); if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } -int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); - taosArrayPush(pMerged->reqs, &pSubmit->data); + taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { + SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pSubmitClone == NULL) { return NULL; } streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->dataRef); } } @@ -135,16 +138,16 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; - SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)elem; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); taosFreeQitem(dst); taosFreeQitem(elem); return (SStreamQueueItem*)pMerged; @@ -162,22 +165,22 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + streamDataSubmitRefDec((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; - int32_t sz = taosArrayGetSize(pMerge->reqs); + SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* dataStr = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(dataStr); + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); taosMemoryFree(pRef); } } - taosArrayDestroy(pMerge->reqs); + taosArrayDestroy(pMerge->submits); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 2c36c299eeba10111aee6bc38d708ffc51ff2322..4e0b0630bc1f08969ebec2cf4a2ff674cb1824ff 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -112,7 +112,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->numOfCols = htonl(numOfCols); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); @@ -189,7 +189,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->compressed = 0; pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; - pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6a83a9a4da6cdc95abe77502539343c46f094a60..85a27f404a53d08290847bf7d646bc521314905d 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -26,17 +26,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; - qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); + const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; + qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + pSubmit->submit.msgLen, pSubmit->submit.ver); + qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; + SArray* blocks = pMerged->submits; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -54,6 +55,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* /*ASSERT(false);*/ qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, terrstr()); + continue; } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { @@ -126,7 +128,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { taosArrayDestroy(pRes); break; } - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -234,7 +236,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { qDebug("stream task %d exec end", pTask->taskId); if (taosArrayGetSize(pRes) != 0) { - SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); streamFreeQitem(input); @@ -244,11 +246,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { qRes->blocks = pRes; if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index fc6e1668ba8f7e8a7def4b86314da40ff184340a..afad78c5e57724809500905e35be136c9354885f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -69,8 +69,8 @@ _err: } void streamMetaClose(SStreamMeta* pMeta) { - tdbCommit(pMeta->db, &pMeta->txn); - tdbPostCommit(pMeta->db, &pMeta->txn); + tdbCommit(pMeta->db, pMeta->txn); + tdbPostCommit(pMeta->db, pMeta->txn); tdbTbClose(pMeta->pTaskDb); tdbTbClose(pMeta->pCheckpointDb); tdbClose(pMeta->db); @@ -115,7 +115,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, goto FAIL; } - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); ASSERT(0); goto FAIL; @@ -152,7 +152,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { tEncodeSStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, &pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { ASSERT(0); return -1; } @@ -220,42 +220,35 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { } int32_t streamMetaBegin(SStreamMeta* pMeta) { - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamMetaCommit(SStreamMeta* pMeta) { - if (tdbCommit(pMeta->db, &pMeta->txn) < 0) { + if (tdbCommit(pMeta->db, pMeta->txn) < 0) { return -1; } - memset(&pMeta->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { + if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) { return -1; } - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamMetaAbort(SStreamMeta* pMeta) { - if (tdbAbort(pMeta->db, &pMeta->txn) < 0) { + if (tdbAbort(pMeta->db, pMeta->txn) < 0) { return -1; } - memset(&pMeta->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pMeta->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < - 0) { - return -1; - } - if (tdbBegin(pMeta->db, &pMeta->txn) < 0) { + + if (tdbBegin(pMeta->db, &pMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index ea31347db5a9470b56e3356455e5308c140781f8..af1d738de05a34b4be237e959996b2c478de5486 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -179,8 +179,8 @@ _err: } void streamStateClose(SStreamState* pState) { - tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn); - tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn); + tdbCommit(pState->pTdbState->db, pState->pTdbState->txn); + tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn); tdbTbClose(pState->pTdbState->pStateDb); tdbTbClose(pState->pTdbState->pFuncStateDb); tdbTbClose(pState->pTdbState->pFillStateDb); @@ -192,71 +192,61 @@ void streamStateClose(SStreamState* pState) { } int32_t streamStateBegin(SStreamState* pState) { - if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - return -1; - } - - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { - tdbTxnClose(&pState->pTdbState->txn); + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + tdbAbort(pState->pTdbState->db, pState->pTdbState->txn); return -1; } return 0; } int32_t streamStateCommit(SStreamState* pState) { - if (tdbCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { - return -1; - } - if (tdbPostCommit(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { + if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { return -1; } - memset(&pState->pTdbState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { return -1; } - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { + + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamStateAbort(SStreamState* pState) { - if (tdbAbort(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { - return -1; - } - memset(&pState->pTdbState->txn, 0, sizeof(TXN)); - if (tdbTxnOpen(&pState->pTdbState->txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, - TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { + if (tdbAbort(pState->pTdbState->db, pState->pTdbState->txn) < 0) { return -1; } - if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn) < 0) { + + if (tdbBegin(pState->pTdbState->db, &pState->pTdbState->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, + TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } return 0; } int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->pTdbState->txn); + return tdbTbUpsert(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, pState->pTdbState->txn); } int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) { return tdbTbGet(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); } int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { - return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), &pState->pTdbState->txn); + return tdbTbDelete(pState->pTdbState->pFuncStateDb, key, sizeof(STupleKey), pState->pTdbState->txn); } // todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->pTdbState->txn); + return tdbTbUpsert(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, pState->pTdbState->txn); } // todo refactor int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->pTdbState->txn); + return tdbTbUpsert(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), value, vLen, pState->pTdbState->txn); } // todo refactor @@ -273,7 +263,7 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal // todo refactor int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { SStateKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), &pState->pTdbState->txn); + return tdbTbDelete(pState->pTdbState->pStateDb, &sKey, sizeof(SStateKey), pState->pTdbState->txn); } int32_t streamStateClear(SStreamState* pState) { @@ -297,7 +287,7 @@ void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number // todo refactor int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { - return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), &pState->pTdbState->txn); + return tdbTbDelete(pState->pTdbState->pFillStateDb, key, sizeof(SWinKey), pState->pTdbState->txn); } int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { @@ -531,7 +521,7 @@ void streamFreeVal(void* val) { tdbFree(val); } int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, const void* value, int32_t vLen) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; return tdbTbUpsert(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), value, vLen, - &pState->pTdbState->txn); + pState->pTdbState->txn); } int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { @@ -554,7 +544,7 @@ int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVa int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key) { SStateSessionKey sKey = {.key = *key, .opNum = pState->number}; - return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), &pState->pTdbState->txn); + return tdbTbDelete(pState->pTdbState->pSessionStateDb, &sKey, sizeof(SStateSessionKey), pState->pTdbState->txn); } SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key) { @@ -833,7 +823,7 @@ _end: int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - &pState->pTdbState->txn); + pState->pTdbState->txn); return 0; } diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 265da9703d959950a0397c19350b9cd88d977767..4dc5f58cfe4aad4d39cf6a66fc4849e5bd6d03cc 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -22,14 +22,12 @@ extern "C" { #include "syncInt.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 5000 -#define ELECT_TIMER_MS_MIN 2500 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) -#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 1000 -#define HEARTBEAT_TICK_NUM 20 +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 5000 +#define ELECT_TIMER_MS_MIN 2500 +#define HEARTBEAT_TIMER_MS 1000 +#define HEARTBEAT_TICK_NUM 20 typedef struct SSyncEnv { uint8_t isStart; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index dee0bf95c7074e1bcbd06cfeb86e9b194a740375..a5524ffbdefd1040c086108ac6a1593c499d6fce 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -215,7 +215,7 @@ int32_t syncNodeStart(SSyncNode* pSyncNode); int32_t syncNodeStartStandBy(SSyncNode* pSyncNode); void syncNodeClose(SSyncNode* pSyncNode); void syncNodePreClose(SSyncNode* pSyncNode); -int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak); +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t *seq); int32_t syncNodeRestore(SSyncNode* pSyncNode); void syncHbTimerDataFree(SSyncHbTimerData* pData); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index cf63df371d7858a1a81e648ae9971bac17bcf99e..3bd94dbab5811fc071d7444ab864d3087f785d53 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -46,6 +46,7 @@ typedef struct SyncClientRequest { uint32_t originalRpcType; // origin RpcMsg msgType uint64_t seqNum; bool isWeak; + int16_t reserved; uint32_t dataLen; // origin RpcMsg.contLen char data[]; // origin RpcMsg.pCont } SyncClientRequest; @@ -56,6 +57,7 @@ typedef struct SyncClientRequestReply { uint32_t msgType; int32_t errCode; SRaftId leaderHint; + int16_t reserved; } SyncClientRequestReply; typedef struct SyncRequestVote { @@ -68,6 +70,7 @@ typedef struct SyncRequestVote { SyncTerm term; SyncIndex lastLogIndex; SyncTerm lastLogTerm; + int16_t reserved; } SyncRequestVote; typedef struct SyncRequestVoteReply { @@ -79,6 +82,7 @@ typedef struct SyncRequestVoteReply { // private data SyncTerm term; bool voteGranted; + int16_t reserved; } SyncRequestVoteReply; typedef struct SyncAppendEntries { @@ -94,6 +98,7 @@ typedef struct SyncAppendEntries { SyncTerm prevLogTerm; SyncIndex commitIndex; SyncTerm privateTerm; + int16_t reserved; uint32_t dataLen; char data[]; } SyncAppendEntries; @@ -111,6 +116,7 @@ typedef struct SyncAppendEntriesReply { SyncIndex matchIndex; SyncIndex lastSendIndex; int64_t startTime; + int16_t reserved; } SyncAppendEntriesReply; typedef struct SyncHeartbeat { @@ -126,6 +132,7 @@ typedef struct SyncHeartbeat { SyncTerm privateTerm; SyncTerm minMatchIndex; int64_t timeStamp; + int16_t reserved; } SyncHeartbeat; typedef struct SyncHeartbeatReply { @@ -140,6 +147,7 @@ typedef struct SyncHeartbeatReply { SyncTerm privateTerm; int64_t startTime; int64_t timeStamp; + int16_t reserved; } SyncHeartbeatReply; typedef struct SyncPreSnapshot { @@ -151,6 +159,7 @@ typedef struct SyncPreSnapshot { // private data SyncTerm term; + int16_t reserved; } SyncPreSnapshot; typedef struct SyncPreSnapshotReply { @@ -163,6 +172,7 @@ typedef struct SyncPreSnapshotReply { // private data SyncTerm term; SyncIndex snapStart; + int16_t reserved; } SyncPreSnapshotReply; typedef struct SyncApplyMsg { @@ -190,6 +200,7 @@ typedef struct SyncSnapshotSend { SSyncCfg lastConfig; int64_t startTime; int32_t seq; + int16_t reserved; uint32_t dataLen; char data[]; } SyncSnapshotSend; @@ -208,6 +219,7 @@ typedef struct SyncSnapshotRsp { int32_t ack; int32_t code; SyncIndex snapBeginIndex; // when ack = SYNC_SNAPSHOT_SEQ_BEGIN, it's valid + int16_t reserved; } SyncSnapshotRsp; typedef struct SyncLeaderTransfer { diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index 4208d40a69584e4fcd7df0e9e4a5f853ed6f62b6..8c7edf85ffa9db4e3ab833c9171f1f388a87fdf4 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -107,7 +107,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode); // private SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex index, bool* pInBuf); int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex); +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 1f9675a3cd41835ea1f6ca20f5a6b21540cd317e..ee8363619266362764152778440b189e9f3ca556 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -44,6 +44,7 @@ typedef struct SSyncSnapshotSender { SyncTerm term; int64_t startTime; int64_t endTime; + int64_t lastSendTime; bool finish; // init when create diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index f198f3809d7eaa307a189487e74077009f64bb4d..be14ef91a40ef6b10c76a5bcf34ca8953e00bed1 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -79,7 +79,6 @@ char* syncUtilPrintBin2(char* ptr, uint32_t len); void syncUtilMsgHtoN(void* msg); void syncUtilMsgNtoH(void* msg); bool syncUtilUserPreCommit(tmsg_t msgType); -bool syncUtilUserCommit(tmsg_t msgType); bool syncUtilUserRollback(tmsg_t msgType); void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 07b1101256a047d7de1442086d605f7b8da2a26c..5fdcbeb91c119bfebb21805b12fc1faa15eb2cab 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -84,7 +84,7 @@ void syncOneReplicaAdvance(SSyncNode* pSyncNode) { } void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { - ASSERT(false && "deprecated"); + ASSERTS(false, "deprecated"); if (pSyncNode == NULL) { sError("pSyncNode is NULL"); return; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f102095ce29247e3b4475885cf06777768236721..6a545424fcee50e8a77affb4e92a13052afd27ae 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -22,8 +22,8 @@ #include "syncEnv.h" #include "syncIndexMgr.h" #include "syncInt.h" -#include "syncPipeline.h" #include "syncMessage.h" +#include "syncPipeline.h" #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncRaftStore.h" @@ -35,6 +35,7 @@ #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" +#include "tglobal.h" #include "tref.h" static void syncNodeEqPingTimer(void* param, void* tmrId); @@ -150,7 +151,7 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } syncNodeStartHeartbeatTimer(pSyncNode); - syncNodeReplicate(pSyncNode); + // syncNodeReplicate(pSyncNode); } syncNodeRelease(pSyncNode); @@ -217,6 +218,26 @@ int32_t syncLeaderTransfer(int64_t rid) { return ret; } +int32_t syncSendTimeoutRsp(int64_t rid, int64_t seq) { + SSyncNode* pNode = syncNodeAcquire(rid); + if (pNode == NULL) return -1; + + SRpcMsg rpcMsg = {0}; + int32_t ret = syncRespMgrGetAndDel(pNode->pSyncRespMgr, seq, &rpcMsg.info); + rpcMsg.code = TSDB_CODE_SYN_TIMEOUT; + + syncNodeRelease(pNode); + if (ret == 1) { + sInfo("send timeout response, seq:%" PRId64 " handle:%p ahandle:%p", seq, rpcMsg.info.handle, + rpcMsg.info.ahandle); + rpcSendResponse(&rpcMsg); + return 0; + } else { + sError("no message handle to send timeout response, seq:%" PRId64, seq); + return -1; + } +} + SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode) { SyncIndex minMatchIndex = SYNC_INDEX_INVALID; @@ -401,62 +422,62 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { bool syncNodeIsReadyForRead(SSyncNode* pSyncNode) { if (pSyncNode == NULL) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; sError("sync ready for read error"); return false; } - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && pSyncNode->restoreFinish) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_NOT_LEADER; + return false; + } + + if (pSyncNode->restoreFinish) { return true; } bool ready = false; - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER && !pSyncNode->restoreFinish) { - if (!pSyncNode->pFsm->FpApplyQueueEmptyCb(pSyncNode->pFsm)) { - // apply queue not empty - ready = false; + if (!pSyncNode->pFsm->FpApplyQueueEmptyCb(pSyncNode->pFsm)) { + // apply queue not empty + ready = false; - } else { - if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { - SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); - SSyncRaftEntry* pEntry = NULL; - SLRUCache* pCache = pSyncNode->pLogStore->pCache; - LRUHandle* h = taosLRUCacheLookup(pCache, &lastIndex, sizeof(lastIndex)); - int32_t code = 0; - if (h) { - pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); - code = 0; + } else { + if (!pSyncNode->pLogStore->syncLogIsEmpty(pSyncNode->pLogStore)) { + SyncIndex lastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SSyncRaftEntry* pEntry = NULL; + SLRUCache* pCache = pSyncNode->pLogStore->pCache; + LRUHandle* h = taosLRUCacheLookup(pCache, &lastIndex, sizeof(lastIndex)); + int32_t code = 0; + if (h) { + pEntry = (SSyncRaftEntry*)taosLRUCacheValue(pCache, h); + code = 0; + + pSyncNode->pLogStore->cacheHit++; + sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", lastIndex, pEntry->bytes, pEntry); - pSyncNode->pLogStore->cacheHit++; - sNTrace(pSyncNode, "hit cache index:%" PRId64 ", bytes:%u, %p", lastIndex, pEntry->bytes, pEntry); + } else { + pSyncNode->pLogStore->cacheMiss++; + sNTrace(pSyncNode, "miss cache index:%" PRId64, lastIndex); - } else { - pSyncNode->pLogStore->cacheMiss++; - sNTrace(pSyncNode, "miss cache index:%" PRId64, lastIndex); + code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, lastIndex, &pEntry); + } - code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, lastIndex, &pEntry); + if (code == 0 && pEntry != NULL) { + if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { + ready = true; } - if (code == 0 && pEntry != NULL) { - if (pEntry->originalRpcType == TDMT_SYNC_NOOP && pEntry->term == pSyncNode->pRaftStore->currentTerm) { - ready = true; - } - - if (h) { - taosLRUCacheRelease(pCache, h, false); - } else { - syncEntryDestroy(pEntry); - } + if (h) { + taosLRUCacheRelease(pCache, h, false); + } else { + syncEntryDestroy(pEntry); } } } } if (!ready) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_NOT_LEADER; - } else { - terrno = TSDB_CODE_APP_NOT_READY; - } + terrno = TSDB_CODE_SYN_RESTORING; } return ready; @@ -537,7 +558,7 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { pMsg->newLeaderId.vgId = pSyncNode->vgId; pMsg->newNodeInfo = newLeader; - int32_t ret = syncNodePropose(pSyncNode, &rpcMsg, false); + int32_t ret = syncNodePropose(pSyncNode, &rpcMsg, false, NULL); rpcFreeCont(rpcMsg.pCont); return ret; } @@ -549,7 +570,11 @@ SSyncState syncGetState(int64_t rid) { if (pSyncNode != NULL) { state.state = pSyncNode->state; state.restored = pSyncNode->restoreFinish; - state.canRead = syncNodeIsReadyForRead(pSyncNode); + if (pSyncNode->vgId != 1) { + state.canRead = syncNodeIsReadyForRead(pSyncNode); + } else { + state.canRead = state.restored; + } syncNodeRelease(pSyncNode); } @@ -665,19 +690,19 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { syncNodeRelease(pSyncNode); } -int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { sError("sync propose error"); return -1; } - int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); + int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak, seq); syncNodeRelease(pSyncNode); return ret; } -int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { +int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; sNError(pSyncNode, "sync propose not leader, %s, type:%s", syncStr(pSyncNode->state), TMSG_INFO(pMsg->msgType)); @@ -734,6 +759,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak) { (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); } + if (seq != NULL) *seq = seqNum; return code; } } @@ -786,9 +812,9 @@ static int32_t syncHbTimerStop(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { } int32_t syncNodeLogStoreRestoreOnNeed(SSyncNode* pNode) { - ASSERT(pNode->pLogStore != NULL && "log store not created"); - ASSERT(pNode->pFsm != NULL && "pFsm not registered"); - ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered"); + ASSERTS(pNode->pLogStore != NULL, "log store not created"); + ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); + ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); SSnapshot snapshot; if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) { sError("vgId:%d, failed to get snapshot info since %s", pNode->vgId, terrstr()); @@ -1020,8 +1046,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // timer ms init pSyncNode->pingBaseLine = PING_TIMER_MS; - pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN; - pSyncNode->hbBaseLine = HEARTBEAT_TIMER_MS; + pSyncNode->electBaseLine = tsElectInterval; + pSyncNode->hbBaseLine = tsHeartbeatInterval; // init ping timer pSyncNode->pPingTimer = NULL; @@ -1111,7 +1137,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->hbrSlowNum = 0; pSyncNode->tmrRoutineNum = 0; - sNTrace(pSyncNode, "sync open, node:%p", pSyncNode); + sNInfo(pSyncNode, "sync open, node:%p", pSyncNode); + sTrace("vgId:%d, tsElectInterval:%d, tsHeartbeatInterval:%d, tsHeartbeatTimeout:%d", pSyncNode->vgId, tsElectInterval, + tsHeartbeatInterval, tsHeartbeatTimeout); return pSyncNode; @@ -1137,8 +1165,8 @@ void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { } int32_t syncNodeRestore(SSyncNode* pSyncNode) { - ASSERT(pSyncNode->pLogStore != NULL && "log store not created"); - ASSERT(pSyncNode->pLogBuf != NULL && "ring log buffer not created"); + ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); + ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore); @@ -1214,6 +1242,26 @@ int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodePreClose(SSyncNode* pSyncNode) { + if (pSyncNode != NULL && pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpApplyQueueItems != NULL) { + while (1) { + int32_t aqItems = pSyncNode->pFsm->FpApplyQueueItems(pSyncNode->pFsm); + sTrace("vgId:%d, pre close, %d items in apply queue", pSyncNode->vgId, aqItems); + if (aqItems == 0 || aqItems == -1) { + break; + } + taosMsleep(20); + } + } + + if (pSyncNode->pNewNodeReceiver != NULL) { + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); + } + + snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); + pSyncNode->pNewNodeReceiver = NULL; + } + // stop elect timer syncNodeStopElectTimer(pSyncNode); @@ -1225,7 +1273,7 @@ void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } void syncNodeClose(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return; - sNTrace(pSyncNode, "sync close, data:%p", pSyncNode); + sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); int32_t ret = raftStoreClose(pSyncNode->pRaftStore); ASSERT(ret == 0); @@ -1812,7 +1860,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { #endif // close receiver - if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + if (pSyncNode != NULL && pSyncNode->pNewNodeReceiver != NULL && + snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { snapshotReceiverForceStop(pSyncNode->pNewNodeReceiver); } @@ -2377,7 +2426,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { continue; } - if (tsNow - recvTime > SYNC_HEART_TIMEOUT_MS) { + if (tsNow - recvTime > tsHeartbeatTimeout) { toCount++; } } @@ -2634,7 +2683,11 @@ int32_t syncNodeOnClientRequest(SSyncNode* ths, SRpcMsg* pMsg, SyncIndex* pRetIn (*pRetIndex) = index; } - return syncNodeAppend(ths, pEntry); + int32_t code = syncNodeAppend(ths, pEntry); + if (code < 0 && ths->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + ASSERTS(false, "failed to append blocking msg"); + } + return code; } return -1; diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f0be976402a95bdbcc0bcfe1de177516851e840d..88727351c67a41ae4f039d009f6ca6d0de7f6120 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "syncPipeline.h" +#include "syncCommit.h" #include "syncIndexMgr.h" #include "syncInt.h" #include "syncRaftEntry.h" @@ -49,7 +50,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt // initial log buffer with at least one item, e.g. commitIndex SSyncRaftEntry* pMatch = pBuf->entries[(index - 1 + pBuf->size) % pBuf->size].pItem; - ASSERT(pMatch != NULL && "no matched log entry"); + ASSERTS(pMatch != NULL, "no matched log entry"); ASSERT(pMatch->index + 1 == index); SSyncLogBufEntry tmp = {.pItem = pEntry, .prevLogIndex = pMatch->index, .prevLogTerm = pMatch->term}; @@ -85,14 +86,14 @@ SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, S if (prevIndex >= pBuf->startIndex) { pEntry = pBuf->entries[(prevIndex + pBuf->size) % pBuf->size].pItem; - ASSERT(pEntry != NULL && "no log entry found"); + ASSERTS(pEntry != NULL, "no log entry found"); prevLogTerm = pEntry->term; return prevLogTerm; } if (pMgr && pMgr->startIndex <= prevIndex && prevIndex < pMgr->endIndex) { int64_t timeMs = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].timeMs; - ASSERT(timeMs != 0 && "no log entry found"); + ASSERTS(timeMs != 0, "no log entry found"); prevLogTerm = pMgr->states[(prevIndex + pMgr->size) % pMgr->size].term; ASSERT(prevIndex == 0 || prevLogTerm != 0); return prevLogTerm; @@ -140,9 +141,9 @@ int32_t syncLogValidateAlignmentOfCommit(SSyncNode* pNode, SyncIndex commitIndex } int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { - ASSERT(pNode->pLogStore != NULL && "log store not created"); - ASSERT(pNode->pFsm != NULL && "pFsm not registered"); - ASSERT(pNode->pFsm->FpGetSnapshotInfo != NULL && "FpGetSnapshotInfo not registered"); + ASSERTS(pNode->pLogStore != NULL, "log store not created"); + ASSERTS(pNode->pFsm != NULL, "pFsm not registered"); + ASSERTS(pNode->pFsm->FpGetSnapshotInfo != NULL, "FpGetSnapshotInfo not registered"); SSnapshot snapshot; if (pNode->pFsm->FpGetSnapshotInfo(pNode->pFsm, &snapshot) < 0) { @@ -310,7 +311,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt ASSERT(pEntry->index == pExist->index); if (pEntry->term != pExist->term) { - (void)syncLogBufferRollback(pBuf, index); + (void)syncLogBufferRollback(pBuf, pNode, index); } else { sTrace("vgId:%d, duplicate log entry received. index: %" PRId64 ", term: %" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -436,12 +437,17 @@ _out: } int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry) { - ASSERT(pFsm->FpCommitCb != NULL && "No commit cb registered for the FSM"); + ASSERTS(pFsm->FpCommitCb != NULL, "No commit cb registered for the FSM"); if ((pNode->replicaNum == 1) && pNode->restoreFinish && pNode->vgId != 1) { return 0; } + if (pNode->vgId != 1 && vnodeIsMsgBlock(pEntry->originalRpcType)) { + sTrace("vgId:%d, blocking msg ready to execute. index:%" PRId64 ", term: %" PRId64 ", type: %s", pNode->vgId, + pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); + } + SRpcMsg rpcMsg = {0}; syncEntry2OriginalRpc(pEntry, &rpcMsg); @@ -457,9 +463,9 @@ int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, Syn cbMeta.flag = -1; (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); ASSERT(rpcMsg.pCont == NULL); - return 0; + return code; } int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf) { @@ -507,17 +513,12 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm if (!syncUtilUserCommit(pEntry->originalRpcType)) { sInfo("vgId:%d, commit sync barrier. index: %" PRId64 ", term:%" PRId64 ", type: %s", vgId, pEntry->index, pEntry->term, TMSG_INFO(pEntry->originalRpcType)); - pBuf->commitIndex = index; - if (!inBuf) { - syncEntryDestroy(pEntry); - pEntry = NULL; - } - continue; } if (syncLogFsmExecute(pNode, pFsm, role, term, pEntry) != 0) { - sError("vgId:%d, failed to execute sync log entry in FSM. log index:%" PRId64 ", term:%" PRId64 "", vgId, - pEntry->index, pEntry->term); + sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 + ", role: %d, current term: %" PRId64, + vgId, pEntry->index, pEntry->term, role, term); goto _out; } pBuf->commitIndex = index; @@ -593,6 +594,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { int count = 0; int64_t firstIndex = -1; SyncTerm term = -1; + int64_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); for (SyncIndex index = pMgr->startIndex; index < pMgr->endIndex; index++) { int64_t pos = index % pMgr->size; @@ -619,7 +621,10 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { retried = true; if (firstIndex == -1) firstIndex = index; - count++; + + if (batchSize <= count++) { + break; + } } ret = 0; @@ -799,8 +804,9 @@ int32_t syncLogReplMgrReplicateProbeOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode int32_t syncLogReplMgrReplicateAttemptedOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); + SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; - int32_t batchSize = TMAX(1, pMgr->size / 20); + int32_t batchSize = TMAX(1, pMgr->size >> (4 + pMgr->retryBackoff)); int32_t count = 0; int64_t nowMs = taosGetMonoTimestampMs(); int64_t limit = pMgr->size >> 1; @@ -899,7 +905,7 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { ASSERT(pNode->logReplMgrs[i] == NULL); pNode->logReplMgrs[i] = syncLogReplMgrCreate(); pNode->logReplMgrs[i]->peerId = i; - ASSERT(pNode->logReplMgrs[i] != NULL && "Out of memory."); + ASSERTS(pNode->logReplMgrs[i] != NULL, "Out of memory."); } return 0; } @@ -971,14 +977,19 @@ void syncLogBufferDestroy(SSyncLogBuffer* pBuf) { return; } -int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex) { +int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex) { ASSERT(pBuf->commitIndex < toIndex && toIndex <= pBuf->endIndex); + sInfo("vgId:%d, rollback sync log buffer. toindex: %" PRId64 ", buffer: [%" PRId64 " %" PRId64 " %" PRId64 + ", %" PRId64 ")", + pNode->vgId, toIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); + + // trunc buffer SyncIndex index = pBuf->endIndex - 1; while (index >= toIndex) { SSyncRaftEntry* pEntry = pBuf->entries[index % pBuf->size].pItem; if (pEntry != NULL) { - syncEntryDestroy(pEntry); + (void)syncEntryDestroy(pEntry); pEntry = NULL; memset(&pBuf->entries[index % pBuf->size], 0, sizeof(pBuf->entries[0])); } @@ -987,6 +998,17 @@ int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SyncIndex toIndex) { pBuf->endIndex = toIndex; pBuf->matchIndex = TMIN(pBuf->matchIndex, index); ASSERT(index + 1 == toIndex); + + // trunc wal + SyncIndex lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + if (lastVer >= toIndex && pNode->pLogStore->syncLogTruncate(pNode->pLogStore, toIndex) < 0) { + sError("vgId:%d, failed to truncate log store since %s. from index:%" PRId64 "", pNode->vgId, terrstr(), toIndex); + return -1; + } + lastVer = pNode->pLogStore->syncLogLastIndex(pNode->pLogStore); + ASSERT(toIndex == lastVer + 1); + + syncLogBufferValidate(pBuf); return 0; } @@ -996,7 +1018,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { ASSERT(lastVer == pBuf->matchIndex); SyncIndex index = pBuf->endIndex - 1; - (void)syncLogBufferRollback(pBuf, pBuf->matchIndex + 1); + (void)syncLogBufferRollback(pBuf, pNode, pBuf->matchIndex + 1); sInfo("vgId:%d, reset sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index cbcf111a85f159d7f90e2de7ed5bad48399ebfba..018ac5bb7da3764d2b54b6dfc7995ffe9d9a47d8 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -41,7 +41,7 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->pCache = taosLRUCacheInit(30 * 1024 * 1024, 1, .5); if (pLogStore->pCache == NULL) { taosMemoryFree(pLogStore); - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -234,18 +234,17 @@ int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncR *ppEntry = NULL; - // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + int64_t ts1 = taosGetTimestampNs(); + taosThreadMutexLock(&(pData->mutex)); + SWalReader* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; sError("vgId:%d, wal handle is NULL", pData->pSyncNode->vgId); - + taosThreadMutexUnlock(&(pData->mutex)); return -1; } - int64_t ts1 = taosGetTimestampNs(); - taosThreadMutexLock(&(pData->mutex)); - int64_t ts2 = taosGetTimestampNs(); code = walReadVer(pWalHandle, index); int64_t ts3 = taosGetTimestampNs(); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 049b02d73e6a89344aa0266fc086cc69db943cb5..79a38cad7a55fdcc0a587c48299d155d3f396931 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -35,11 +35,16 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { pObj->seqNum = 0; taosThreadMutexInit(&(pObj->mutex), NULL); + SSyncNode *pNode = pObj->data; + sTrace("vgId:%d, create resp manager", pNode->vgId); return pObj; } void syncRespMgrDestroy(SSyncRespMgr *pObj) { if (pObj != NULL) { + SSyncNode *pNode = pObj->data; + sTrace("vgId:%d, destroy resp manager", pNode->vgId); + taosThreadMutexLock(&pObj->mutex); taosHashCleanup(pObj->pRespHash); taosThreadMutexUnlock(&pObj->mutex); @@ -81,6 +86,8 @@ int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t seq, SRespStub *pStub) { taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object + } else { + sNError(pObj->data, "get message handle, no object of seq:%" PRIu64, seq); } taosThreadMutexUnlock(&pObj->mutex); @@ -99,6 +106,8 @@ int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t seq, SRpcHandleInfo *p taosThreadMutexUnlock(&pObj->mutex); return 1; // get one object + } else { + sNError(pObj->data, "get-and-del message handle, no object of seq:%" PRIu64, seq); } taosThreadMutexUnlock(&pObj->mutex); @@ -114,7 +123,7 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { SArray *delIndexArray = taosArrayInit(4, sizeof(uint64_t)); if (delIndexArray == NULL) return; - sDebug("vgId:%d, resp mgr begin clean by ttl", pSyncNode->vgId); + sDebug("vgId:%d, resp manager begin clean by ttl", pSyncNode->vgId); while (pStub) { size_t len; void *key = taosHashGetKey(pStub, &len); @@ -143,34 +152,39 @@ static void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) { // TODO: and make rpcMsg body, call commit cb // pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &pStub->rpcMsg, cbMeta); - - pStub->rpcMsg.code = TSDB_CODE_SYN_NOT_LEADER; - if (pStub->rpcMsg.info.handle != NULL) { - tmsgSendRsp(&pStub->rpcMsg); - } + SRpcMsg rpcMsg = {.info = pStub->rpcMsg.info, .code = TSDB_CODE_SYN_TIMEOUT}; + sInfo("vgId:%d, message handle:%p expired, type:%s ahandle:%p", pSyncNode->vgId, rpcMsg.info.handle, + TMSG_INFO(pStub->rpcMsg.msgType), rpcMsg.info.ahandle); + rpcSendResponse(&rpcMsg); } pStub = taosHashIterate(pObj->pRespHash, pStub); } int32_t arraySize = taosArrayGetSize(delIndexArray); - sDebug("vgId:%d, resp mgr end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize); + sDebug("vgId:%d, resp manager end clean by ttl, sum:%d, cnt:%d, array-size:%d", pSyncNode->vgId, sum, cnt, arraySize); for (int32_t i = 0; i < arraySize; ++i) { uint64_t *pSeqNum = taosArrayGet(delIndexArray, i); taosHashRemove(pObj->pRespHash, pSeqNum, sizeof(uint64_t)); - sDebug("vgId:%d, resp mgr clean by ttl, seq:%" PRId64 "", pSyncNode->vgId, *pSeqNum); + sDebug("vgId:%d, resp manager clean by ttl, seq:%" PRId64, pSyncNode->vgId, *pSeqNum); } taosArrayDestroy(delIndexArray); } void syncRespCleanRsp(SSyncRespMgr *pObj) { + SSyncNode *pNode = pObj->data; + sTrace("vgId:%d, clean all rsp", pNode->vgId); + taosThreadMutexLock(&pObj->mutex); syncRespCleanByTTL(pObj, -1, true); taosThreadMutexUnlock(&pObj->mutex); } void syncRespClean(SSyncRespMgr *pObj) { + SSyncNode *pNode = pObj->data; + sTrace("vgId:%d, clean rsp by ttl", pNode->vgId); + taosThreadMutexLock(&pObj->mutex); syncRespCleanByTTL(pObj, pObj->ttl, false); taosThreadMutexUnlock(&pObj->mutex); diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a316d78cdb0b730859d9076c6d46c49aba41eaab..540f40a4c018c435348552c3da88f8e243bfa72c 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -103,6 +103,7 @@ int32_t snapshotSenderStart(SSyncSnapshotSender *pSender) { pSender->sendingMS = 0; pSender->term = pSender->pSyncNode->pRaftStore->currentTerm; pSender->startTime = taosGetTimestampMs(); + pSender->lastSendTime = pSender->startTime; pSender->finish = false; // build begin msg @@ -201,6 +202,8 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); + pSender->lastSendTime = taosGetTimestampMs(); + // event log if (pSender->seq == SYNC_SNAPSHOT_SEQ_END) { sSTrace(pSender, "snapshot sender finish"); @@ -213,32 +216,35 @@ int32_t snapshotSend(SSyncSnapshotSender *pSender) { // send snapshot data from cache int32_t snapshotReSend(SSyncSnapshotSender *pSender) { // send current block data - if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { - // build msg - SRpcMsg rpcMsg = {0}; - (void)syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId); - - SyncSnapshotSend *pMsg = rpcMsg.pCont; - pMsg->srcId = pSender->pSyncNode->myRaftId; - pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; - pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; - pMsg->beginIndex = pSender->snapshotParam.start; - pMsg->lastIndex = pSender->snapshot.lastApplyIndex; - pMsg->lastTerm = pSender->snapshot.lastApplyTerm; - pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; - pMsg->lastConfig = pSender->lastConfig; - pMsg->seq = pSender->seq; + // build msg + SRpcMsg rpcMsg = {0}; + (void)syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId); + + SyncSnapshotSend *pMsg = rpcMsg.pCont; + pMsg->srcId = pSender->pSyncNode->myRaftId; + pMsg->destId = (pSender->pSyncNode->replicasId)[pSender->replicaIndex]; + pMsg->term = pSender->pSyncNode->pRaftStore->currentTerm; + pMsg->beginIndex = pSender->snapshotParam.start; + pMsg->lastIndex = pSender->snapshot.lastApplyIndex; + pMsg->lastTerm = pSender->snapshot.lastApplyTerm; + pMsg->lastConfigIndex = pSender->snapshot.lastConfigIndex; + pMsg->lastConfig = pSender->lastConfig; + pMsg->seq = pSender->seq; + + if (pSender->pCurrentBlock != NULL && pSender->blockLen > 0) { // pMsg->privateTerm = pSender->privateTerm; memcpy(pMsg->data, pSender->pCurrentBlock, pSender->blockLen); + } + + // send msg + syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); + syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); - // send msg - syncNodeSendMsgById(&pMsg->destId, pSender->pSyncNode, &rpcMsg); - syncLogSendSyncSnapshotSend(pSender->pSyncNode, pMsg, ""); + pSender->lastSendTime = taosGetTimestampMs(); - // event log - sSTrace(pSender, "snapshot sender resend"); - } + // event log + sSTrace(pSender, "snapshot sender resend"); return 0; } @@ -339,6 +345,8 @@ bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceive void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { + // event log + sRTrace(pReceiver, "snapshot receiver force stop"); int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, &(pReceiver->snapshot)); ASSERT(ret == 0); @@ -348,7 +356,7 @@ void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { pReceiver->start = false; // event log - sRTrace(pReceiver, "snapshot receiver force stop"); + // sRTrace(pReceiver, "snapshot receiver force stop"); } int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { @@ -548,6 +556,7 @@ _START_RECEIVER: sNTrace(pSyncNode, "snapshot receiver pre waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, pMsg->startTime); taosMsleep(10); + timeNow = taosGetTimestampMs(); } if (snapshotReceiverIsStart(pReceiver)) { @@ -668,6 +677,7 @@ static int32_t syncNodeOnSnapshotEnd(SSyncNode *pSyncNode, SyncSnapshotSend *pMs sNTrace(pSyncNode, "snapshot receiver finish waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, pMsg->startTime); taosMsleep(10); + timeNow = taosGetTimestampMs(); } int32_t code = snapshotReceiverFinish(pReceiver, pMsg); @@ -883,9 +893,9 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // receive ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { snapshotSenderStop(pSender, true); - SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); if (pMgr) { - syncLogReplMgrReset(pMgr); + syncLogReplMgrReset(pMgr); } return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index cf933d98b23f15efca0867464857aa083e6f846e..16e593d0e49695f3ec7ac02d47388106bcb9d281 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -19,6 +19,8 @@ #include "syncRaftCfg.h" #include "syncRaftLog.h" #include "syncReplication.h" +#include "syncRespMgr.h" +#include "syncSnapshot.h" #include "syncUtil.h" static void syncNodeCleanConfigIndex(SSyncNode* ths) { @@ -69,6 +71,20 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { } int64_t timeNow = taosGetTimestampMs(); + + for (int i = 0; i < ths->peersNum; ++i) { + SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(ths->peersId[i])); + if (pSender != NULL) { + if (ths->isStart && ths->state == TAOS_SYNC_STATE_LEADER && pSender->start && + timeNow - pSender->lastSendTime > SYNC_SNAP_RESEND_MS) { + snapshotReSend(pSender); + } else { + sTrace("vgId:%d, do not resend: nstart%d, now:%" PRId64 ", lstsend:%" PRId64 ", diff:%" PRId64, ths->vgId, + ths->isStart, timeNow, pSender->lastSendTime, timeNow - pSender->lastSendTime); + } + } + } + if (atomic_load_64(&ths->snapshottingIndex) != SYNC_INDEX_INVALID) { // end timeout wal snapshot if (timeNow - ths->snapshottingTime > SYNC_DEL_WAL_MS && @@ -85,11 +101,9 @@ static int32_t syncNodeTimerRoutine(SSyncNode* ths) { } } -#if 0 if (!syncNodeIsMnode(ths)) { syncRespClean(ths->pSyncRespMgr); } -#endif return 0; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 7787438dfe0850929f7472d3ee1e4338b22a80f8..a034e6ad836024c65b510f25031c340e959342d2 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -160,8 +160,6 @@ void syncUtilMsgNtoH(void* msg) { bool syncUtilUserPreCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } -bool syncUtilUserCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } - bool syncUtilUserRollback(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; } void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) { @@ -568,7 +566,7 @@ void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* p syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-snapshot-send from %s:%d {term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 ", lterm:%" PRId64 + "send sync-snapshot-send to %s:%d {term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64 ", seq:%d}, %s", host, port, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime, pMsg->seq, s); } @@ -595,7 +593,7 @@ void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMs syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); sNTrace(pSyncNode, - "send sync-snapshot-rsp from %s:%d {term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 + "send sync-snapshot-rsp to %s:%d {term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64 ", lterm:%" PRId64 ", stime:%" PRId64 ", ack:%d}, %s", host, port, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime, pMsg->ack, s); } diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index 1eb803e846133035ddc78122acab6cb82195c18d..e1a8a5c8329a0ee320282fcfc5481a2f46a68c2c 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -19,7 +19,7 @@ SyncAppendEntriesReply *createMsg() { pMsg->success = true; pMsg->matchIndex = 77; pMsg->term = 33; - pMsg->privateTerm = 44; + // pMsg->privateTerm = 44; pMsg->startTime = taosGetTimestampMs(); return pMsg; } diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 057f2ea6dd7f38241fcc0759113e65e513d020d3..7a5d0777bb0ba00f3d5f4507088138d8c776fe15 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -337,7 +337,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); - int32_t ret = syncPropose(rid, pRpcMsg, false); + int32_t ret = syncPropose(rid, pRpcMsg, false, NULL); if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { diff --git a/source/libs/sync/test/syncConfigChangeTest.cpp b/source/libs/sync/test/syncConfigChangeTest.cpp index bab3d2236f3a33cd40bf356cb60227a6731ef022..f35a23b15b3d49ecbbaad922f3bb005ec7f22865 100644 --- a/source/libs/sync/test/syncConfigChangeTest.cpp +++ b/source/libs/sync/test/syncConfigChangeTest.cpp @@ -249,7 +249,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); - int32_t ret = syncPropose(rid, pRpcMsg, false); + int32_t ret = syncPropose(rid, pRpcMsg, false, NULL); if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { diff --git a/source/libs/sync/test/syncReplicateTest.cpp b/source/libs/sync/test/syncReplicateTest.cpp index 4a82bba15d959f545a333d84a4af8ed7b62e0225..22132eb01f68d92449bcd5d9b808a7e480fbb39c 100644 --- a/source/libs/sync/test/syncReplicateTest.cpp +++ b/source/libs/sync/test/syncReplicateTest.cpp @@ -189,7 +189,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); - int32_t ret = syncPropose(rid, pRpcMsg, false); + int32_t ret = syncPropose(rid, pRpcMsg, false, NULL); if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader", s, alreadySend); } else { diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 9ae79e11fb9dd100c3c101025da86380d192a748..7b636085f211d0caae7450340889687cdd8abcb9 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,5 +1,5 @@ #include "syncTest.h" -#include +// #include /* typedef enum { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 8c486df118f26b53264d8830b9d3bfc8292adb14..4bc2e92d0c7a55ac6dd5d3c6aa697acad3e57d53 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -396,7 +396,7 @@ int main(int argc, char** argv) { if (alreadySend < writeRecordNum) { SRpcMsg* pRpcMsg = createRpcMsg(alreadySend, writeRecordNum, myIndex); - int32_t ret = syncPropose(rid, pRpcMsg, false); + int32_t ret = syncPropose(rid, pRpcMsg, false, NULL); if (ret == -1 && terrno == TSDB_CODE_SYN_NOT_LEADER) { sTrace("%s value%d write not leader, leaderTransferWait:%d", simpleStr, alreadySend, leaderTransferWait); } else { diff --git a/source/libs/sync/test/sync_test_lib/inc/syncIO.h b/source/libs/sync/test/sync_test_lib/inc/syncIO.h index 19f896182e11fa3793d7b4396cb83f806456e597..98b013b46d167e6936658e841f05a7304d7c4642 100644 --- a/source/libs/sync/test/sync_test_lib/inc/syncIO.h +++ b/source/libs/sync/test/sync_test_lib/inc/syncIO.h @@ -81,6 +81,8 @@ int32_t syncIOQTimerStop(); int32_t syncIOPingTimerStart(); int32_t syncIOPingTimerStop(); +void syncEntryDestory(SSyncRaftEntry* pEntry); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/test/sync_test_lib/src/syncIO.c b/source/libs/sync/test/sync_test_lib/src/syncIO.c index 2c244517132c75ed75417acf282d8d2f39e5d22e..4b305f823ff3a1002c4e8c29ed0c3afe02c371a7 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncIO.c +++ b/source/libs/sync/test/sync_test_lib/src/syncIO.c @@ -97,7 +97,7 @@ int32_t syncIOEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { syncRpcMsgLog2(logBuf, pMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); STaosQueue *pMsgQ = gSyncIO->pMsgQ; @@ -381,7 +381,7 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg); SSyncIO *io = pParent; SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); taosWriteQitem(io->pMsgQ, pTemp); } @@ -441,7 +441,7 @@ static void syncIOTickQ(void *param, void *tmrId) { SRpcMsg rpcMsg; syncPingReply2RpcMsg(pMsg, &rpcMsg); SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0); memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg); taosWriteQitem(io->pMsgQ, pTemp); @@ -469,3 +469,5 @@ static void syncIOTickPing(void *param, void *tmrId) { taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); } + +void syncEntryDestory(SSyncRaftEntry* pEntry) {} \ No newline at end of file diff --git a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c index 1ea7629601a4a7dc9846266e6652b3b4caf5b06a..3b6007df6bd346986a7e6a7b46ddb0300761d2fc 100644 --- a/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c +++ b/source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c @@ -1583,8 +1583,8 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); cJSON_AddItemToObject(pRoot, "destId", pDestId); - snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); - cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + // snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + // cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); cJSON_AddStringToObject(pRoot, "term", u64buf); diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 7ab2bc399579ffd5dc07dc81628a9da85ce9bbad..10a99bb1fab525a9ff4569d55c3d688bb099ee46 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -17,6 +17,7 @@ #define _TD_TDB_H_ #include "os.h" +#include "tdbOs.h" #ifdef __cplusplus extern "C" { @@ -33,7 +34,8 @@ typedef struct STxn TXN; // TDB int32_t tdbOpen(const char *dbname, int szPage, int pages, TDB **ppDb, int8_t rollback); int32_t tdbClose(TDB *pDb); -int32_t tdbBegin(TDB *pDb, TXN *pTxn); +int32_t tdbBegin(TDB *pDb, TXN **pTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags); int32_t tdbCommit(TDB *pDb, TXN *pTxn); int32_t tdbPostCommit(TDB *pDb, TXN *pTxn); int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn); @@ -77,12 +79,18 @@ int32_t tdbTxnClose(TXN *pTxn); // other void tdbFree(void *); +typedef struct hashset_st *hashset_t; + +void hashset_destroy(hashset_t set); + struct STxn { int flags; int64_t txnId; void *(*xMalloc)(void *, size_t); void (*xFree)(void *, void *); - void *xArg; + void *xArg; + tdb_fd_t jfd; + hashset_t jPageSet; }; // error code diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6a60073426a106a1922b783bce29cce49290daa9..5c1f264460630d12249cafa5ea20f8df5e279976 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -69,7 +69,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); -int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv, SBTree **ppBt) { SBTree *pBt; int ret; @@ -106,22 +106,29 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg if (pgno == 0) { // fetch page & insert into main db SPage *pPage; - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + TXN *txn; - pPager->inTran = 1; + ret = tdbBegin(pEnv, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); + if (ret < 0) { + tdbOsFree(pBt); + return -1; + } SBtreeInitPageArg zArg; zArg.flags = 0x1 | 0x2; // root leaf node; zArg.pBt = pBt; - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn); + ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, txn); if (ret < 0) { + tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } ret = tdbPagerWrite(pPager, pPage); if (ret < 0) { tdbError("failed to write page since %s", terrstr()); + tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } @@ -130,18 +137,19 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg pBt->info.nLevel = 1; pBt->info.nData = 0; pBt->tbname = (char *)tbname; - // ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pgno, sizeof(SPgno), &txn); - ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), &txn); + + ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pBt->info, sizeof(pBt->info), txn); if (ret < 0) { + tdbAbort(pEnv, txn); + tdbOsFree(pBt); return -1; } } - // tdbUnrefPage(pPage); - tdbPCacheRelease(pPager->pCache, pPage, &txn); - tdbCommit(pPager->pEnv, &txn); - tdbPostCommit(pPager->pEnv, &txn); - tdbTxnClose(&txn); + tdbPCacheRelease(pPager->pCache, pPage, txn); + + tdbCommit(pPager->pEnv, txn); + tdbPostCommit(pPager->pEnv, txn); } ASSERT(pgno != 0); @@ -1534,10 +1542,21 @@ int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn) { memset(&pBtc->coder, 0, sizeof(SCellDecoder)); if (pTxn == NULL) { - pBtc->pTxn = &pBtc->txn; - tdbTxnOpen(pBtc->pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); + if (!pTxn) { + return -1; + } + + if (tdbTxnOpen(pTxn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + tdbOsFree(pTxn); + return -1; + } + + pBtc->pTxn = pTxn; + pBtc->freeTxn = 1; } else { pBtc->pTxn = pTxn; + pBtc->freeTxn = 0; } return 0; @@ -2231,6 +2250,10 @@ int tdbBtcClose(SBTC *pBtc) { tdbFree(pBtc->coder.pVal); } + if (pBtc->freeTxn) { + tdbTxnClose(pBtc->pTxn); + } + return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 5aff5b7bb2d649d0caad06433419dd19b4f34013..c79279c658a746e83a40eaed6fd6cb8cbe6e31d7 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -99,19 +99,37 @@ int tdbClose(TDB *pDb) { int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); } -int32_t tdbBegin(TDB *pDb, TXN *pTxn) { +int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, + int flags) { SPager *pPager; int ret; + int64_t txnId = ++pDb->txnId; + if (txnId == INT64_MAX) { + pDb->txnId = 0; + } + + TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn)); + if (!pTxn) { + return -1; + } + + if (tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags) < 0) { + tdbOsFree(pTxn); + return -1; + } for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) { ret = tdbPagerBegin(pPager, pTxn); if (ret < 0) { tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName, pTxn->txnId); + tdbTxnClose(pTxn); return -1; } } + *ppTxn = pTxn; + return 0; } @@ -144,6 +162,8 @@ int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) { } } + tdbTxnClose(pTxn); + return 0; } @@ -176,6 +196,8 @@ int32_t tdbAbort(TDB *pDb, TXN *pTxn) { } } + tdbTxnClose(pTxn); + return 0; } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 425e494ab682db3cf1aa798890b7741c557d7538..b67fe562eb4806ced4e862a0b7185566189a0432 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -184,9 +184,9 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage, nRef); if (pPage) { - tdbDebug("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); } else { - tdbDebug("pcache/fetch page %p", pPage); + tdbTrace("pcache/fetch page %p", pPage); } return pPage; @@ -202,7 +202,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); - tdbDebug("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + tdbTrace("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); if (nRef == 0) { // test the nRef again to make sure // it is safe th handle the page @@ -221,8 +221,6 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { // } } tdbPCacheUnlock(pCache); - // printf("thread %" PRId64 " relas page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, - // TDB_PAGE_PGNO(pPage), pPage, nRef); } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; } @@ -335,8 +333,7 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable--; - // printf("pin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); } } @@ -349,7 +346,7 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { ASSERT(pPage->pLruNext == NULL); - tdbDebug("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages); + tdbTrace("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages); if (pPage->id < pCache->nPages) { pPage->pLruPrev = &(pCache->lru); pPage->pLruNext = pCache->lru.pLruNext; @@ -359,9 +356,9 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable++; // printf("unpin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); } else { - tdbDebug("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); tdbPCacheRemovePageFromHash(pCache, pPage); tdbPageDestroy(pPage, tdbDefaultFree, NULL); @@ -381,7 +378,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { // printf("rmv page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); } - tdbDebug("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); + tdbTrace("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); } static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { @@ -392,8 +389,7 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { pCache->nPage++; - // printf("add page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); + tdbTrace("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); } static int tdbPCacheOpenImpl(SPCache *pCache) { diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 016ad65cf3849ec7a1af762853932df4e5943d83..ac2725d8ec670f73c56ea99983d2247dfd742bf9 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -69,18 +69,19 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbDebug("page/create: %p %p", pPage, xMalloc); + tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbDebug("page/destroy: %p %p", pPage, xFree); + tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); + ASSERT(!pPage->isDirty); ASSERT(xFree); for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { - tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); + tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); tdbOsFree(pPage->apOvfl[iOvfl]); } @@ -91,7 +92,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) } void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { - tdbDebug("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); + tdbTrace("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); @@ -108,7 +109,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { - tdbDebug("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); + tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); @@ -153,7 +154,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pCell, szCell); - tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); + tdbTrace("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; @@ -203,7 +204,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (pPage->aiOvfl[iOvfl] == idx) { // remove the over flow cell tdbOsFree(pPage->apOvfl[iOvfl]); - tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); + tdbTrace("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); for (; (++iOvfl) < pPage->nOverflow;) { pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; @@ -256,7 +257,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); - tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); + tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } pToPage->apOvfl[iOvfl] = pNewCell; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index ea67986da222fc65c2ae3fcc3998a35ec94c5127..648e99d6a53343abc5479577e66200aa85dd2a50 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -142,7 +142,7 @@ int hashset_contains(hashset_t set, void *item) { static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage *, void *, int), void *arg, u8 loadPage); static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); -static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); +static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage); static FORCE_INLINE int32_t pageCmpFn(const SRBTreeNode *lhs, const SRBTreeNode *rhs) { SPage *pPageL = (SPage *)(((uint8_t *)lhs) - offsetof(SPage, node)); @@ -219,85 +219,27 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { int tdbPagerClose(SPager *pPager) { if (pPager) { + /* if (pPager->inTran) { tdbOsClose(pPager->jfd); } + */ tdbOsClose(pPager->fd); tdbOsFree(pPager); } return 0; } -/* -int tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt) { - SPgno pgno; - SPage *pPage; - int ret; - - if (pPager->dbOrigSize > 0) { - pgno = 1; - } else { - pgno = 0; - } - { - // TODO: try to search the main DB to get the page number - // pgno = 0; - } - - if (pgno == 0 && toCreate) { - // allocate a new child page - TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); - - pPager->inTran = 1; - - SBtreeInitPageArg zArg; - zArg.flags = 0x1 | 0x2; // root leaf node; - zArg.pBt = pBt; - ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn); - if (ret < 0) { - return -1; - } - - // ret = tdbPagerAllocPage(pPager, &pPage, &pgno); - // if (ret < 0) { - // return -1; - //} - - // TODO: Need to zero the page - - ret = tdbPagerWrite(pPager, pPage); - if (ret < 0) { - tdbError("failed to write page since %s", terrstr()); - return -1; - } - - tdbTxnClose(&txn); - } - - *ppgno = pgno; - return 0; -} -*/ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - ASSERT(pPager->inTran); -#if 0 - if (pPager->inTran == 0) { - ret = tdbPagerBegin(pPager); - if (ret < 0) { - return -1; - } - } -#endif - + // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release tdbRefPage(pPage); - tdbDebug("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); // Set page as dirty pPage->isDirty = 1; @@ -322,15 +264,16 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Write page to journal if neccessary if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize && - (pPager->jPageSet == NULL || !hashset_contains(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { + (pPager->pActiveTxn->jPageSet == NULL || + !hashset_contains(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))))) { ret = tdbPagerWritePageToJournal(pPager, pPage); if (ret < 0) { tdbError("failed to write page to journal since %s", tstrerror(terrno)); return -1; } - if (pPager->jPageSet) { - hashset_add(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + if (pPager->pActiveTxn->jPageSet) { + hashset_add(pPager->pActiveTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } } @@ -338,23 +281,28 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { } int tdbPagerBegin(SPager *pPager, TXN *pTxn) { + /* if (pPager->inTran) { return 0; } - + */ // Open the journal - pPager->jfd = tdbOsOpen(pPager->jFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); - if (TDB_FD_INVALID(pPager->jfd)) { + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + pTxn->jfd = tdbOsOpen(jTxnFileName, TDB_O_CREAT | TDB_O_RDWR, 0755); + if (TDB_FD_INVALID(pTxn->jfd)) { tdbError("failed to open file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - pPager->jPageSet = hashset_create(); - // TODO: write the size of the file + pTxn->jPageSet = hashset_create(); + pPager->pActiveTxn = pTxn; + // TODO: write the size of the file + /* pPager->inTran = 1; - + */ return 0; } @@ -363,9 +311,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { int ret; // sync the journal file - ret = tdbOsFSync(pPager->jfd); + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync: %s. jFileName:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -377,7 +325,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage = (SPage *)pNode; ASSERT(pPage->nOverflow == 0); - ret = tdbPagerWritePageToDB(pPager, pPage); + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -395,8 +343,8 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - if (pPager->jPageSet) { - hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + if (pTxn->jPageSet) { + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); } tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -415,35 +363,36 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { } int tdbPagerPostCommit(SPager *pPager, TXN *pTxn) { + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + // remove the journal file - if (tdbOsClose(pPager->jfd) < 0) { - tdbError("failed to close jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + if (tdbOsClose(pTxn->jfd) < 0) { + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. file:%s", strerror(errno), pPager->jFileName); + if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - if (pPager->jPageSet) { - hashset_destroy(pPager->jPageSet); - } - pPager->inTran = 0; + // pPager->inTran = 0; return 0; } int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { SPage *pPage; + SPgno maxPgno = pPager->dbOrigSize; int ret; // sync the journal file - ret = tdbOsFSync(pPager->jfd); + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. jFileName:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -454,7 +403,12 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; if (pPage->isLocal) continue; - ret = tdbPagerWritePageToDB(pPager, pPage); + + SPgno pgno = TDB_PAGE_PGNO(pPage); + if (pgno > maxPgno) { + maxPgno = pgno; + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -462,7 +416,8 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { } tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize); - pPager->dbOrigSize = pPager->dbFileSize; + pPager->dbOrigSize = maxPgno; + // pPager->dbOrigSize = pPager->dbFileSize; // release the page iter = tRBTreeIterCreate(&pPager->rbt, 1); @@ -474,6 +429,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) { tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } + /* tdbTrace("reset dirty tree: %p", &pPager->rbt); tRBTreeCreate(&pPager->rbt, pageCmpFn); @@ -495,15 +451,15 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { SPgno journalSize = 0; int ret; - // 0, sync the journal file - ret = tdbOsFSync(pPager->jfd); + // sync the journal file + ret = tdbOsFSync(pTxn->jfd); if (ret < 0) { - tdbError("failed to fsync jfd due to %s. file:%s", strerror(errno), pPager->jFileName); + tdbError("failed to fsync jfd: %s. jfile:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - tdb_fd_t jfd = pPager->jfd; + tdb_fd_t jfd = pTxn->jfd; ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); if (ret < 0) { @@ -567,7 +523,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); - hashset_remove(pPager->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); + hashset_remove(pTxn->jPageSet, (void *)((long)TDB_PAGE_PGNO(pPage))); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -575,11 +531,22 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { tRBTreeCreate(&pPager->rbt, pageCmpFn); // 4, remove the journal file - tdbOsClose(pPager->jfd); - (void)tdbOsRemove(pPager->jFileName); - hashset_destroy(pPager->jPageSet); + if (tdbOsClose(pTxn->jfd) < 0) { + tdbError("failed to close jfd: %s. file:%s, %" PRId64, strerror(errno), pPager->jFileName, pTxn->txnId); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } - pPager->inTran = 0; + char jTxnFileName[TDB_FILENAME_LEN]; + sprintf(jTxnFileName, "%s.%" PRId64, pPager->jFileName, pTxn->txnId); + + if (tdbOsRemove(jTxnFileName) < 0 && errno != ENOENT) { + tdbError("failed to remove file due to %s. file:%s", strerror(errno), jTxnFileName); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + // pPager->inTran = 0; return 0; } @@ -604,7 +571,7 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) { if (pgno > maxPgno) { maxPgno = pgno; } - ret = tdbPagerWritePageToDB(pPager, pPage); + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); return -1; @@ -753,14 +720,16 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage pgno = TDB_PAGE_PGNO(pPage); - tdbTrace("tdbttl init pager:%p, pgno:%d, loadPage:%d, size:%d", pPager, pgno, loadPage, pPager->dbOrigSize); + tdbTrace("tdb/pager:%p, pgno:%d, loadPage:%d, size:%d", pPager, pgno, loadPage, pPager->dbOrigSize); if (loadPage && pgno <= pPager->dbOrigSize) { init = 1; nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); - tdbTrace("tdbttl pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); + tdbTrace("tdb/pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32, pPager, pgno, nRead, pPage->pageSize); + TDB_UNLOCK_PAGE(pPage); return -1; } } else { @@ -802,17 +771,18 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { pgno = TDB_PAGE_PGNO(pPage); - ret = tdbOsWrite(pPager->jfd, &pgno, sizeof(pgno)); + ret = tdbOsWrite(pPager->pActiveTxn->jfd, &pgno, sizeof(pgno)); if (ret < 0) { - tdbError("failed to write pgno due to %s. file:%s, pgno:%u", strerror(errno), pPager->jFileName, pgno); + tdbError("failed to write pgno due to %s. file:%s, pgno:%u, txnId:%" PRId64, strerror(errno), pPager->jFileName, + pgno, pPager->pActiveTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - ret = tdbOsWrite(pPager->jfd, pPage->pData, pPage->pageSize); + ret = tdbOsWrite(pPager->pActiveTxn->jfd, pPage->pData, pPage->pageSize); if (ret < 0) { - tdbError("failed to write page data due to %s. file:%s, pageSize:%ld", strerror(errno), pPager->jFileName, - (long)pPage->pageSize); + tdbError("failed to write page data due to %s. file:%s, pageSize:%d, txnId:%" PRId64, strerror(errno), + pPager->jFileName, pPage->pageSize, pPager->pActiveTxn->txnId); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -820,13 +790,6 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { return 0; } /* -struct TdFile { - TdThreadRwlock rwlock; - int refId; - int fd; - FILE *fp; -} TdFile; -*/ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { i64 offset; int ret; @@ -846,16 +809,32 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { return -1; } - // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset); + return 0; +} +*/ +static int tdbPagerPWritePageToDB(SPager *pPager, SPage *pPage) { + i64 offset; + int ret; + + offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); + + ret = tdbOsPWrite(pPager->fd, pPage->pData, pPage->pageSize, offset); + if (ret < 0) { + tdbError("failed to pwrite page data due to %s. file:%s, pageSize:%d", strerror(errno), pPager->dbFileName, + pPage->pageSize); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + return 0; } -int tdbPagerRestore(SPager *pPager, SBTree *pBt) { +static int tdbPagerRestore(SPager *pPager, SBTree *pBt, const char *jFileName) { int ret = 0; SPgno journalSize = 0; u8 *pageBuf = NULL; - tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); + tdb_fd_t jfd = tdbOsOpen(jFileName, TDB_O_RDWR, 0755); if (jfd == NULL) { return 0; } @@ -928,12 +907,54 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return 0; } +int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt) { + tdbDirEntryPtr pDirEntry; + tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); + if (pDir == NULL) { + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); + return -1; + } + + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { + char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); + if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { + if (tdbPagerRestore(pPager, pBt, name) < 0) { + tdbCloseDir(&pDir); + + tdbError("failed to restore file due to %s. jFileName:%s", strerror(errno), name); + return -1; + } + } + } + + tdbCloseDir(&pDir); + + return 0; +} + int tdbPagerRollback(SPager *pPager) { - if (tdbOsRemove(pPager->jFileName) < 0 && errno != ENOENT) { - tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), pPager->jFileName); - terrno = TAOS_SYSTEM_ERROR(errno); + tdbDirEntryPtr pDirEntry; + tdbDirPtr pDir = taosOpenDir(pPager->pEnv->dbName); + if (pDir == NULL) { + tdbError("failed to open %s since %s", pPager->pEnv->dbName, strerror(errno)); return -1; } + while ((pDirEntry = tdbReadDir(pDir)) != NULL) { + char *name = tdbDirEntryBaseName(tdbGetDirEntryName(pDirEntry)); + + if (strncmp(TDB_MAINDB_NAME "-journal", name, 16) == 0) { + if (tdbOsRemove(name) < 0 && errno != ENOENT) { + tdbCloseDir(&pDir); + + tdbError("failed to remove file due to %s. jFileName:%s", strerror(errno), name); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + } + + tdbCloseDir(&pDir); + return 0; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 8b029b06d6d6b0c0332b48bb36075fc9e3fbf28d..c5c2d6aebe0af02e52a1d52154ca84722e36c878 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -108,7 +108,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF ASSERT(pPager != NULL); // pTb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); + ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, pEnv, &(pTb->pBt)); if (ret < 0) { tdbOsFree(pTb); return -1; @@ -117,7 +117,7 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF if (rollback) { tdbPagerRollback(pPager); } else { - ret = tdbPagerRestore(pPager, pTb->pBt); + ret = tdbPagerRestoreJournals(pPager, pTb->pBt); if (ret < 0) { tdbOsFree(pTb); return -1; diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index f173d897793ad1b0df0224b882b7369394b02696..055d9c7f984b05003a7a69d996e1eb54a51bbe8c 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -28,4 +28,15 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void return 0; } -int tdbTxnClose(TXN *pTxn) { return 0; } \ No newline at end of file +int tdbTxnClose(TXN *pTxn) { + if (pTxn) { + if (pTxn->jPageSet) { + hashset_destroy(pTxn->jPageSet); + pTxn->jPageSet = NULL; + } + + tdbOsFree(pTxn); + } + + return 0; +} diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 731b1927e70382680119afbc64d333e4e841a8e7..055a8a1062362bec646004c0aad8fae277f67fc1 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -147,11 +147,11 @@ struct SBTC { SPage *pgStack[BTREE_MAX_DEPTH + 1]; SCellDecoder coder; TXN *pTxn; - TXN txn; + i8 freeTxn; }; // SBTree -int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv, SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); @@ -197,7 +197,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); -int tdbPagerRestore(SPager *pPager, SBTree *pBt); +int tdbPagerRestoreJournals(SPager *pPager, SBTree *pBt); int tdbPagerRollback(SPager *pPager); // tdbPCache.c ==================================== @@ -382,26 +382,24 @@ struct STDB { #ifdef USE_MAINDB TTB *pMainDb; #endif + int64_t txnId; }; -typedef struct hashset_st *hashset_t; - struct SPager { char *dbFileName; char *jFileName; int pageSize; uint8_t fid[TDB_FILE_ID_LEN]; tdb_fd_t fd; - tdb_fd_t jfd; SPCache *pCache; SPgno dbFileSize; SPgno dbOrigSize; - //SPage *pDirty; - hashset_t jPageSet; - SRBTree rbt; - u8 inTran; - SPager *pNext; // used by TDB - SPager *pHashNext; // used by TDB + // SPage *pDirty; + SRBTree rbt; + // u8 inTran; + TXN *pActiveTxn; + SPager *pNext; // used by TDB + SPager *pHashNext; // used by TDB #ifdef USE_MAINDB TDB *pEnv; #endif diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index b5dd27052c6f1cb79f04f7eec3b4ee38f83dc085..db4283e26778098a75e98e6e349f373eb8fdd039 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -47,15 +47,21 @@ typedef TdFilePtr tdb_fd_t; #define TDB_O_RDWR (TD_FILE_WRITE) | (TD_FILE_READ) #define tdbOsOpen(PATH, OPTION, MODE) taosOpenFile((PATH), (OPTION)) - -#define tdbOsClose(FD) taosCloseFile(&(FD)) -#define tdbOsRead taosReadFile -#define tdbOsPRead taosPReadFile -#define tdbOsWrite taosWriteFile -#define tdbOsFSync taosFsyncFile -#define tdbOsLSeek taosLSeekFile -#define tdbOsRemove remove -#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL) +#define tdbOsClose(FD) taosCloseFile(&(FD)) +#define tdbOsRead taosReadFile +#define tdbOsPRead taosPReadFile +#define tdbOsWrite taosWriteFile +#define tdbOsPWrite taosPWriteFile +#define tdbOsFSync taosFsyncFile +#define tdbOsLSeek taosLSeekFile +#define tdbDirPtr TdDirPtr +#define tdbDirEntryPtr TdDirEntryPtr +#define tdbReadDir taosReadDir +#define tdbGetDirEntryName taosGetDirEntryName +#define tdbDirEntryBaseName taosDirEntryBaseName +#define tdbCloseDir taosCloseDir +#define tdbOsRemove remove +#define tdbOsFileSize(FD, PSIZE) taosFStatFile(FD, PSIZE, NULL) /* directory */ #define tdbOsMkdir taosMkDir diff --git a/source/libs/tdb/test/tdbExOVFLTest.cpp b/source/libs/tdb/test/tdbExOVFLTest.cpp index 305e91f62c19f1d067c92fd2ed298dfdd1ee0f70..b16bc643d3687e00b04cfdcdc9f544e56cd0d0c4 100644 --- a/source/libs/tdb/test/tdbExOVFLTest.cpp +++ b/source/libs/tdb/test/tdbExOVFLTest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "tlog.h" typedef struct SPoolMem { int64_t size; @@ -170,11 +171,9 @@ static void insertOfp(void) { SPoolMem *pPool = openPool(); // start a transaction - TXN txn; - int64_t txnid = 0; - ++txnid; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + TXN *txn = NULL; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // generate value payload // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) @@ -185,12 +184,12 @@ static void insertOfp(void) { // insert the generated big data // char const *key = "key1"; char const *key = "key123456789"; - ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); } // TEST(TdbOVFLPagesTest, DISABLED_TbInsertTest) { @@ -258,11 +257,9 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) { SPoolMem *pPool = openPool(); // start a transaction - TXN txn; - int64_t txnid = 0; - ++txnid; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + TXN *txn; + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // generate value payload // char val[((4083 - 4 - 3 - 2) + 1) * 100]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) @@ -271,7 +268,7 @@ TEST(TdbOVFLPagesTest, TbDeleteTest) { generateBigVal(val, valLen); { // insert the generated big data - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn); + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); } @@ -297,7 +294,7 @@ tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_REA tdbBegin(pEnv, &txn); */ { // upsert the data - ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), &txn); + ret = tdbTbUpsert(pDb, "key1", strlen("key1"), "value1", strlen("value1"), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -316,7 +313,7 @@ tdbBegin(pEnv, &txn); } { // delete the data - ret = tdbTbDelete(pDb, "key1", strlen("key1"), &txn); + ret = tdbTbDelete(pDb, "key1", strlen("key1"), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -335,8 +332,8 @@ tdbBegin(pEnv, &txn); } // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); } // TEST(tdb_test, DISABLED_simple_insert1) { @@ -346,7 +343,7 @@ TEST(tdb_test, simple_insert1) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1; - TXN txn; + TXN *txn; int const pageSize = 4096; taosRemoveDir("tdb"); @@ -365,16 +362,13 @@ TEST(tdb_test, simple_insert1) { // char val[(4083 - 4 - 3 - 2)]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) char val[(4083 - 4 - 3 - 2) + 1]; // pSize(4096) - amSize(1) - pageHdr(8) - footerSize(4) int64_t poolLimit = 4096; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key0"); @@ -393,26 +387,25 @@ TEST(tdb_test, simple_insert1) { val[i] = c; } - ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, &txn); + ret = tdbTbInsert(pDb, "key1", strlen("key1"), val, valLen, txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one if (pPool->size >= poolLimit) { // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // start a new transaction clearPool(pPool); - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); { // Query the data void *pVal = NULL; diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f3a301cf5bed77918f1c4d208f146769688b5206..cd02eb8d5ef9471bf0cfa36d3dbe4727b0186567 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "tlog.h" typedef struct SPoolMem { int64_t size; @@ -125,7 +126,7 @@ TEST(tdb_test, DISABLED_simple_insert1) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -142,40 +143,35 @@ TEST(tdb_test, DISABLED_simple_insert1) { char key[64]; char val[64]; int64_t poolLimit = 4096; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); // if pool is full, commit the transaction and start a new one if (pPool->size >= poolLimit) { // commit current transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // start a new transaction clearPool(pPool); - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); { // Query the data void *pVal = NULL; @@ -245,7 +241,7 @@ TEST(tdb_test, DISABLED_simple_insert2) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -261,21 +257,18 @@ TEST(tdb_test, DISABLED_simple_insert2) { { char key[64]; char val[64]; - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -312,8 +305,8 @@ TEST(tdb_test, DISABLED_simple_insert2) { } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); ret = tdbTbDrop(pDb); GTEST_ASSERT_EQ(ret, 0); @@ -331,7 +324,7 @@ TEST(tdb_test, DISABLED_simple_delete1) { TTB *pDb; char key[128]; char data[128]; - TXN txn; + TXN *txn; TDB *pEnv; SPoolMem *pPool; void *pKey = NULL; @@ -353,14 +346,13 @@ TEST(tdb_test, DISABLED_simple_delete1) { ret = tdbTbOpen("db.db", -1, -1, tKeyCmpr, pEnv, &pDb, 0); GTEST_ASSERT_EQ(ret, 0); - tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); // loop to insert batch data for (int iData = 0; iData < nKV; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -378,7 +370,7 @@ TEST(tdb_test, DISABLED_simple_delete1) { for (int iData = nKV - 1; iData > 30; iData--) { sprintf(key, "key%d", iData); - ret = tdbTbDelete(pDb, key, strlen(key), &txn); + ret = tdbTbDelete(pDb, key, strlen(key), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -413,7 +405,8 @@ TEST(tdb_test, DISABLED_simple_delete1) { tdbTbcClose(pDbc); - tdbCommit(pEnv, &txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); closePool(pPool); @@ -430,7 +423,7 @@ TEST(tdb_test, DISABLED_simple_upsert1) { char data[64]; void *pData = NULL; SPoolMem *pPool; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -444,13 +437,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) { pPool = openPool(); // insert some data - tdbTxnOpen(&txn, 0, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -467,11 +459,12 @@ TEST(tdb_test, DISABLED_simple_upsert1) { for (int iData = 0; iData < nData; iData++) { sprintf(key, "key%d", iData); sprintf(data, "data%d-u", iData); - ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), &txn); + ret = tdbTbUpsert(pDb, key, strlen(key), data, strlen(data), txn); GTEST_ASSERT_EQ(ret, 0); } - tdbCommit(pEnv, &txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // query the data for (int iData = 0; iData < nData; iData++) { @@ -492,7 +485,7 @@ TEST(tdb_test, multi_thread_query) { TTB *pDb; tdb_cmpr_fn_t compFunc; int nData = 1000000; - TXN txn; + TXN *txn; taosRemoveDir("tdb"); @@ -508,26 +501,18 @@ TEST(tdb_test, multi_thread_query) { char key[64]; char val[64]; int64_t poolLimit = 4096 * 20; // 1M pool limit - int64_t txnid = 0; SPoolMem *pPool; // open the pool pPool = openPool(); // start a transaction - txnid++; - txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED; - txn.txnId = -1; - txn.xMalloc = poolMalloc; - txn.xFree = poolFree; - txn.xArg = pPool; - // tdbTxnOpen(&txn, txnid, poolMalloc, poolFree, pPool, ); - tdbBegin(pEnv, &txn); + tdbBegin(pEnv, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); - ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), &txn); + ret = tdbTbInsert(pDb, key, strlen(key), val, strlen(val), txn); GTEST_ASSERT_EQ(ret, 0); } @@ -578,7 +563,7 @@ TEST(tdb_test, multi_thread_query) { std::vector threads; for (int i = 0; i < nThreads; i++) { if (i == 0) { - threads.push_back(std::thread(tdbCommit, pEnv, &txn)); + threads.push_back(std::thread(tdbCommit, pEnv, txn)); } else { threads.push_back(std::thread(f, pDb, nData)); } @@ -589,8 +574,8 @@ TEST(tdb_test, multi_thread_query) { } // commit the transaction - tdbCommit(pEnv, &txn); - tdbTxnClose(&txn); + tdbCommit(pEnv, txn); + tdbPostCommit(pEnv, txn); // Close a database tdbTbClose(pDb); @@ -621,17 +606,12 @@ TEST(tdb_test, DISABLED_multi_thread1) { GTEST_ASSERT_EQ(ret, 0); auto insert = [](TDB *pDb, TTB *pTb, int nData, int *stop, std::shared_timed_mutex *mu) { - TXN txn = {0}; + TXN *txn = NULL; char key[128]; char val[128]; SPoolMem *pPool = openPool(); - txn.flags = TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED; - txn.txnId = -1; - txn.xMalloc = poolMalloc; - txn.xFree = poolFree; - txn.xArg = pPool; - tdbBegin(pDb, &txn); + tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); for (int iData = 1; iData <= nData; iData++) { sprintf(key, "key%d", iData); sprintf(val, "value%d", iData); @@ -644,14 +624,17 @@ TEST(tdb_test, DISABLED_multi_thread1) { } if (pPool->size > 1024 * 1024) { - tdbCommit(pDb, &txn); + tdbCommit(pDb, txn); + tdbPostCommit(pDb, txn); clearPool(pPool); - tdbBegin(pDb, &txn); + tdbBegin(pDb, &txn, poolMalloc, poolFree, pPool, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); } } - tdbCommit(pDb, &txn); + tdbCommit(pDb, txn); + tdbPostCommit(pDb, txn); + closePool(pPool); *stop = 1; diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 833937aa41ecdcd4045241105ce34836baf11e59..2db4a72795566d3a52d55650d275fe9c65c54ae5 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -47,20 +47,22 @@ typedef struct { char label[TSDB_LABEL_LEN]; char user[TSDB_UNI_LEN]; // meter ID - int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size - int8_t encryption; // encrypt or not - int32_t retryLimit; // retry limit - int32_t retryInterval; // retry interval ms + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not int32_t retryMinInterval; // retry init interval int32_t retryStepFactor; // retry interval factor int32_t retryMaxInterval; // retry max interval int32_t retryMaxTimouet; + int32_t failFastThreshold; + int32_t failFastInterval; + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); bool (*retry)(int32_t code, tmsg_t msgType); bool (*startTimer)(int32_t code, tmsg_t msgType); void (*destroyFp)(void* ahandle); + bool (*failFastFp)(tmsg_t msgType); int index; void* parent; diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index 559715084cea3cd89ff77a9b934d7e18d0c5072a..95bc532994d7581eb538e9ec22d3d93395f16424 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -54,8 +54,6 @@ void tmsgSendRsp(SRpcMsg* pMsg) { #endif } -void tmsgSendRedirectRsp(SRpcMsg* pMsg, const SEpSet* pNewEpSet) { (*defaultMsgCb.sendRedirectRspFp)(pMsg, pNewEpSet); } - void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLinkArgFp)(pMsg); } void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index c6a5cfdc953d775215d7bb29f3e10e87bfa854df..55c3c61b05ebb0728858b8151f5fc33425fc7a65 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -47,20 +47,26 @@ void* rpcOpen(const SRpcInit* pInit) { } pRpc->compressSize = pInit->compressSize; + if (pRpc->compressSize < 0) { + pRpc->compressSize = -1; + } + pRpc->encryption = pInit->encryption; - pRpc->retryLimit = pInit->retryLimit; - pRpc->retryInterval = pInit->retryInterval; pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval pRpc->retryStepFactor = pInit->retryStepFactor; pRpc->retryMaxInterval = pInit->retryMaxInterval; pRpc->retryMaxTimouet = pInit->retryMaxTimouet; + pRpc->failFastThreshold = pInit->failFastThreshold; + pRpc->failFastInterval = pInit->failFastInterval; + // register callback handle pRpc->cfp = pInit->cfp; pRpc->retry = pInit->rfp; pRpc->startTimer = pInit->tfp; pRpc->destroyFp = pInit->dfp; + pRpc->failFastFp = pInit->ffp; pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; if (pRpc->numOfThreads <= 0) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a9afbd7ba85ecca5e6951173c66dea43e55f3725..6ad126162eb88f3979e5e9f3990d96d8552c78ee 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -84,6 +84,8 @@ typedef struct SCliThrd { SHashObj* fqdn2ipCache; SCvtAddr cvtAddr; + SHashObj* failFastCache; + SCliMsg* stopMsg; bool quit; @@ -96,6 +98,13 @@ typedef struct SCliObj { SCliThrd** pThreadObj; } SCliObj; +typedef struct { + int32_t reinit; + int64_t timestamp; + int32_t count; + int32_t threshold; + int64_t interval; +} SFailFastItem; // conn pool // add expire timeout and capacity limit static void* createConnPool(int size); @@ -276,6 +285,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } destroyCmsg(msg); } + memset(&conn->ctx, 0, sizeof(conn->ctx)); } bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { @@ -580,6 +590,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { } static int32_t allocConnRef(SCliConn* conn, bool update) { if (update) { + transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; } @@ -593,6 +604,7 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { if (update) { + transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; } @@ -688,6 +700,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); QUEUE_REMOVE(&conn->q); QUEUE_INIT(&conn->q); + transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); conn->refId = -1; @@ -722,6 +735,7 @@ static void cliDestroy(uv_handle_t* handle) { conn->timer = NULL; } + transReleaseExHandle(transGetRefMgt(), conn->refId); transRemoveExHandle(transGetRefMgt(), conn->refId); taosMemoryFree(conn->ip); taosMemoryFree(conn->stream); @@ -853,7 +867,7 @@ void cliSend(SCliConn* pConn) { int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); if (status != 0) { - tGError("%s conn %p failed to sent msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), + tGError("%s conn %p failed to send msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), uv_err_name(status)); cliHandleExcept(pConn); } @@ -863,7 +877,6 @@ _RETURN: } void cliConnCb(uv_connect_t* req, int status) { - // impl later SCliConn* pConn = req->data; SCliThrd* pThrd = pConn->hostThrd; @@ -875,7 +888,33 @@ void cliConnCb(uv_connect_t* req, int status) { } if (status != 0) { - tError("%s conn %p failed to connect server:%s", CONN_GET_INST_LABEL(pConn), pConn, uv_strerror(status)); + SCliMsg* pMsg = transQueueGet(&pConn->cliMsgs, 0); + STrans* pTransInst = pThrd->pTransInst; + + tError("%s msg %s failed to send, conn %p failed to connect to %s:%d, reason: %s", CONN_GET_INST_LABEL(pConn), + pMsg ? TMSG_INFO(pMsg->msg.msgType) : 0, pConn, pConn->ip, pConn->port, uv_strerror(status)); + if (pMsg != NULL && REQUEST_NO_RESP(&pMsg->msg) && + (pTransInst->failFastFp != NULL && pTransInst->failFastFp(pMsg->msg.msgType))) { + char* ip = pConn->ip; + uint32_t port = pConn->port; + char key[TSDB_FQDN_LEN + 64] = {0}; + CONN_CONSTRUCT_HASH_KEY(key, ip, port); + + SFailFastItem* item = taosHashGet(pThrd->failFastCache, key, strlen(key)); + int64_t cTimestamp = taosGetTimestampMs(); + if (item != NULL) { + int32_t elapse = cTimestamp - item->timestamp; + if (elapse >= 0 && elapse <= pTransInst->failFastInterval) { + item->count++; + } else { + item->count = 1; + item->timestamp = cTimestamp; + } + } else { + SFailFastItem item = {.count = 1, .timestamp = cTimestamp}; + taosHashPut(pThrd->failFastCache, key, strlen(key), &item, sizeof(SFailFastItem)); + } + } cliHandleExcept(pConn); return; } @@ -918,7 +957,6 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = exh->handle; transReleaseExHandle(transGetRefMgt(), refId); - tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); if (T_REF_VAL_GET(conn) == 2) { @@ -1018,15 +1056,31 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - char tbuf[256] = {0}; - EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - if (!EPSET_IS_VALID(&pCtx->epSet)) { tError("invalid epset"); destroyCmsg(pMsg); return; } + if (REQUEST_NO_RESP(&pMsg->msg) && (pTransInst->failFastFp != NULL && pTransInst->failFastFp(pMsg->msg.msgType))) { + char* ip = EPSET_GET_INUSE_IP(&pCtx->epSet); + uint32_t port = EPSET_GET_INUSE_PORT(&pCtx->epSet); + char key[TSDB_FQDN_LEN + 64] = {0}; + CONN_CONSTRUCT_HASH_KEY(key, ip, port); + + SFailFastItem* item = taosHashGet(pThrd->failFastCache, key, strlen(key)); + if (item != NULL) { + int32_t elapse = (int32_t)(taosGetTimestampMs() - item->timestamp); + if (item->count >= pTransInst->failFastThreshold && (elapse >= 0 && elapse <= pTransInst->failFastInterval)) { + STraceId* trace = &(pMsg->msg.info.traceId); + tGTrace("%s, msg %s cancel to send, reason: failed to connect %s:%d: count: %d, at %d", pTransInst->label, + TMSG_INFO(pMsg->msg.msgType), ip, port, item->count, elapse); + destroyCmsg(pMsg); + return; + } + } + } + bool ignore = false; SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); if (ignore == true) { @@ -1299,6 +1353,8 @@ static SCliThrd* createThrdObj(void* trans) { pThrd->destroyAhandleFp = pTransInst->destroyFp; pThrd->fqdn2ipCache = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pThrd->failFastCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pThrd->quit = false; return pThrd; } @@ -1325,6 +1381,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosHashCleanup(pThrd->fqdn2ipCache); + taosHashCleanup(pThrd->failFastCache); taosMemoryFree(pThrd); } @@ -1437,18 +1494,35 @@ FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { bool cliResetEpset(STransConnCtx* pCtx, STransMsg* pResp, bool hasEpSet) { bool noDelay = true; if (hasEpSet == false) { - // assert(pResp->contLen == 0); if (pResp->contLen == 0) { if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { noDelay = false; } else { EPSET_FORWARD_INUSE(&pCtx->epSet); } - } else { - if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { - noDelay = false; + } else if (pResp->contLen != 0) { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } } else { - EPSET_FORWARD_INUSE(&pCtx->epSet); + if (!transEpSetIsEqual(&pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset"); + pCtx->epSet = epSet; + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet.numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(&pCtx->epSet); + } + } } } } else { @@ -1500,6 +1574,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryStep = 0; pCtx->retryInit = true; pCtx->retryCode = TSDB_CODE_SUCCESS; + + // already retry, not use handle specified by app; + pMsg->msg.info.handle = 0; } if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { @@ -1519,14 +1596,16 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transFreeMsg(pResp->pCont); transUnrefCliHandle(pConn); } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || - code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_VND_STOPPED) { + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING || code == TSDB_CODE_VND_STOPPED) { tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); noDelay = cliResetEpset(pCtx, pResp, true); transFreeMsg(pResp->pCont); addConnToPool(pThrd->pool, pConn); } else if (code == TSDB_CODE_SYN_RESTORING) { tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); - noDelay = cliResetEpset(pCtx, pResp, false); + noDelay = cliResetEpset(pCtx, pResp, true); addConnToPool(pThrd->pool, pConn); transFreeMsg(pResp->pCont); } else { @@ -1550,9 +1629,9 @@ bool cliGenRetryRule(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->retryNextInterval = pCtx->retryMaxInterval; } - if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { - return false; - } + // if (-1 != pCtx->retryMaxTimeout && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + // return false; + // } } else { pCtx->retryNextInterval = 0; pCtx->epsetRetryCnt++; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ad8d57c97a7a8bc9f078ab2171d5a4a76d4f46b3..09f9a78ab88ecce3b95441d9bb0727be6c18e2fd 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -282,6 +282,9 @@ void transCtxCleanup(STransCtx* ctx) { } void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (src->args == NULL || src->freeFunc == NULL) { + return; + } if (dst->args == NULL) { dst->args = src->args; dst->brokenVal = src->brokenVal; diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 71e2e3051179d1f51e1d64300f763e2712cd9161..2b1f68d5f64952cbd933e3c20c1c507c2f2be966 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -250,9 +250,9 @@ static bool uvHandleReq(SSvrConn* pConn) { transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, transMsg.code, (int)(cost)); } else { - tGWarn("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", - transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, - transMsg.code, (int)(cost)); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, resp:%d, code:%d, cost:%dus", + transLabel(pTransInst), pConn, TMSG_INFO(transMsg.msgType), pConn->dst, pConn->src, msgLen, pHead->noResp, + transMsg.code, (int)(cost)); } } @@ -1195,6 +1195,8 @@ void transCloseServer(void* arg) { sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } + } else { + uv_loop_close(srv->loop); } taosMemoryFree(srv->pThreadObj); diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 464559c1e05b6ef1325ae428f6d55d8e30c6f63d..4e2395b17bc2cc65e258d77a5907e56a538634b5 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -128,7 +128,7 @@ void *processShellMsg(void *arg) { void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { SRpcMsg *pTemp; - pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM); + pTemp = taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); int32_t idx = balance % multiQ->numOfThread; diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index d77acbbb6f8930250543f7433422aec8527cda79..8e6628bb21eabc25e2e66000e58b796854c5ac34 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -24,7 +24,9 @@ bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) { return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver; } -bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; } +bool FORCE_INLINE walIsEmpty(SWal* pWal) { + return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) +} int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } @@ -105,7 +107,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { void* ptr = taosMemoryRealloc(buf, capacity); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } buf = ptr; @@ -632,7 +634,7 @@ int walRollFileInfo(SWal* pWal) { // TODO: change to emplace back SWalFileInfo* pNewInfo = taosMemoryMalloc(sizeof(SWalFileInfo)); if (pNewInfo == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -664,7 +666,7 @@ char* walMetaSerialize(SWal* pWal) { if (pFiles) { cJSON_Delete(pFiles); } - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -895,7 +897,7 @@ int walLoadMeta(SWal* pWal) { int size = (int)fileSize; char* buf = taosMemoryMalloc(size + 5); if (buf == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memset(buf, 0, size + 5); diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 5d0f020b02cfc62a23ffbf3de2dc656e7d150a9b..702d05f57696c55e88b94d2d335674bd8f262c49 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -121,7 +121,16 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { - wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); + wError("vgId:%d, failed to init taosArray of fileInfoSet due to %s. path:%s", pWal->cfg.vgId, strerror(errno), + pWal->path); + goto _err; + } + + // init gc + pWal->toDeleteFiles = taosArrayInit(8, sizeof(SWalFileInfo)); + if (pWal->toDeleteFiles == NULL) { + wError("vgId:%d, failed to init taosArray of toDeleteFiles due to %s. path:%s", pWal->cfg.vgId, strerror(errno), + pWal->path); goto _err; } @@ -168,7 +177,7 @@ _err: } int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { - if (pWal == NULL) return TSDB_CODE_WAL_APP_ERROR; + if (pWal == NULL) return TSDB_CODE_APP_ERROR; if (pWal->cfg.level == pCfg->level && pWal->cfg.fsyncPeriod == pCfg->fsyncPeriod) { wDebug("vgId:%d, old walLevel:%d fsync:%d, new walLevel:%d fsync:%d not change", pWal->cfg.vgId, pWal->cfg.level, @@ -203,6 +212,8 @@ void walClose(SWal *pWal) { pWal->pIdxFile = NULL; taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; + taosArrayDestroy(pWal->toDeleteFiles); + pWal->toDeleteFiles = NULL; void *pIter = NULL; while (1) { @@ -212,6 +223,7 @@ void walClose(SWal *pWal) { taosMemoryFree(pRef); } taosHashCleanup(pWal->pRefHash); + pWal->pRefHash = NULL; taosThreadMutexUnlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1350ca0c37616c2f748a01a87de26b77b3c36823..4c3fb3c0ba1d6081aaca8179a8c31c6725279b20 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -48,7 +48,7 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pHead = taosMemoryMalloc(sizeof(SWalCkHead)); if (pReader->pHead == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pReader); return NULL; } @@ -271,57 +271,58 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { return 0; } -static int32_t walFetchBodyNew(SWalReader *pRead) { - SWalCont *pReadHead = &pRead->pHead->head; +static int32_t walFetchBodyNew(SWalReader *pReader) { + SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (pReader->capacity < pReadHead->bodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pRead->pHead = ptr; - pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pReader->pHead = ptr; + pReadHead = &pReader->pHead->head; + pReader->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno)); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; + pReader->curInvalid = 1; ASSERT(0); return -1; } if (pReadHead->version != ver) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pRead->pWal->cfg.vgId, - pRead->pHead->head.version, ver); - pRead->curInvalid = 1; + wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64, pReader->pWal->cfg.vgId, + pReader->pHead->head.version, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; ASSERT(0); return -1; } - if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; + if (walValidBodyCksum(pReader->pHead) != 0) { + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; ASSERT(0); return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pRead->pWal->cfg.vgId, ver); - pRead->curVersion = ver + 1; + wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + pReader->curVersion = ver + 1; return 0; } @@ -437,7 +438,7 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { if (pRead->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(*ppHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } *ppHead = ptr; @@ -489,7 +490,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { int32_t code; bool seeked = false; - if (pReader->pWal->vers.firstVer == -1) { + if (walIsEmpty(pReader->pWal)) { terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } @@ -546,7 +547,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReader->pHead->head.bodyLen); if (ptr == NULL) { - terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; taosThreadMutexUnlock(&pReader->mutex); return -1; } diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index eae5d9f1a7f7e1b4624270e2e74f91581b1e9541..e86111109ce4fc9768be8b662670ae711f935e39 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -32,15 +32,18 @@ SWalRef *walOpenRef(SWal *pWal) { return pRef; } -#if 1 void walCloseRef(SWal *pWal, int64_t refId) { SWalRef **ppRef = taosHashGet(pWal->pRefHash, &refId, sizeof(int64_t)); if (ppRef == NULL) return; SWalRef *pRef = *ppRef; + if (pRef) { + wDebug("vgId:%d, wal close ref %" PRId64 ", refId %" PRId64, pWal->cfg.vgId, pRef->refVer, pRef->refId); + } else { + wDebug("vgId:%d, wal close ref null, refId %" PRId64, pWal->cfg.vgId, refId); + } taosHashRemove(pWal->pRefHash, &refId, sizeof(int64_t)); taosMemoryFree(pRef); } -#endif int32_t walRefVer(SWalRef *pRef, int64_t ver) { SWal *pWal = pRef->pWal; diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2dde0de39b31a17db73a4ba70fa929116a7c60cb..a5c7bf1abdb8fd4ee71beb7fb3186a21cda03c32 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -120,9 +120,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } - // delete files + // delete files in descending order int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); - for (int i = pWal->writeCur + 1; i < fileSetSize; i++) { + for (int i = fileSetSize - 1; i >= pWal->writeCur + 1; i--) { walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr); taosRemoveFile(fnameStr); @@ -217,14 +217,9 @@ int32_t walRollback(SWal *pWal, int64_t ver) { pWal->vers.lastVer = ver - 1; if (pWal->vers.lastVer < pWal->vers.firstVer) { ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1); - pWal->vers.firstVer = -1; } ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; - if (((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer < ver - 1) { - ASSERT(((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize == 0); - ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->firstVer = -1; - } taosCloseFile(&pIdxFile); taosCloseFile(&pLogFile); @@ -338,6 +333,7 @@ int32_t walEndSnapshot(SWal *pWal) { } else { wDebug("vgId:%d, wal no remove", pWal->cfg.vgId); } + // iterate files, until the searched result for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { wDebug("vgId:%d, wal check remove file %" PRId64 "(file size %" PRId64 " close ts %" PRId64 @@ -350,34 +346,17 @@ int32_t walEndSnapshot(SWal *pWal) { wDebug("vgId:%d, check pass", pWal->cfg.vgId); deleteCnt++; newTotSize -= iter->fileSize; + taosArrayPush(pWal->toDeleteFiles, iter); } wDebug("vgId:%d, check not pass", pWal->cfg.vgId); } - wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt); - int32_t actualDelete = 0; - char fnameStr[WAL_FILE_LEN]; - // remove file - for (int i = 0; i < deleteCnt; i++) { - pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); - wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); - if (taosRemoveFile(fnameStr) < 0) { - goto UPDATE_META; - } - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); - wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); - if (taosRemoveFile(fnameStr) < 0) { - ASSERT(0); - } - actualDelete++; - } UPDATE_META: // make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, actualDelete); + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->writeCur = -1; - pWal->vers.firstVer = -1; + pWal->vers.firstVer = pWal->vers.lastVer + 1; } else { pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } @@ -392,6 +371,26 @@ int32_t walEndSnapshot(SWal *pWal) { goto END; } + // delete files + deleteCnt = taosArrayGetSize(pWal->toDeleteFiles); + wDebug("vgId:%d, wal should delete %d files", pWal->cfg.vgId, deleteCnt); + char fnameStr[WAL_FILE_LEN]; + for (int i = 0; i < deleteCnt; i++) { + pInfo = taosArrayGet(pWal->toDeleteFiles, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); + wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); + if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to remove log file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); + goto END; + } + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); + if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { + ASSERT(0); + } + } + taosArrayClear(pWal->toDeleteFiles); + END: taosThreadMutexUnlock(&pWal->mutex); return code; @@ -489,9 +488,7 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); ASSERT(pFileInfo != NULL); - if (pFileInfo->firstVer == -1) { - pFileInfo->firstVer = index; - } + ASSERT(pFileInfo->firstVer != -1); pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -527,7 +524,10 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } // set status - if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; + if (pWal->vers.firstVer == -1) { + ASSERT(index == 0); + pWal->vers.firstVer = 0; + } pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + bodyLen; pFileInfo->lastVer = index; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 421901184bb4bd1ae953a9941764c37605f1370a..331d24174591648a080242b22b889866efa93eb4 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -163,7 +163,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } *pos = TD_DIRSEP[0]; @@ -179,7 +179,7 @@ int32_t taosMulMkDir(const char *dirname) { code = mkdir(temp, 0755); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } } @@ -225,7 +225,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } *pos = TD_DIRSEP[0]; @@ -241,7 +241,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { code = mkdir(temp, mode); #endif if (code < 0 && errno != EEXIST) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return code; } } @@ -394,8 +394,8 @@ char *taosDirEntryBaseName(char *name) { char *pPoint = strrchr(name, '/'); if (pPoint != NULL) { if (*(pPoint + 1) == '\0') { - *pPoint = '\0'; - return taosDirEntryBaseName(name); + *pPoint = '\0'; + return taosDirEntryBaseName(name); } return pPoint + 1; } @@ -497,3 +497,12 @@ int32_t taosCloseDir(TdDirPtr *ppDir) { return 0; #endif } + +void taosGetCwd(char *buf, int32_t len) { +#if !defined(WINDOWS) + char *unused __attribute__((unused)); + unused = getcwd(buf, len - 1); +#else + strncpy(buf, "not implemented on windows", len - 1); +#endif +} diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7063d1f5745b09097f76fe3f4fd12e3f3a693bec..c4627ffb7559a045ffae9a32e9a01a87a0e32de9 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -37,7 +37,7 @@ float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; char *tsProcPath = NULL; -char tsSIMDEnable = 0; +char tsSIMDBuiltins = 0; char tsSSE42Enable = 0; char tsAVXEnable = 0; char tsAVX2Enable = 0; diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index c3283ffe849f029fe4fb3b81a9c4c778b49da6b6..d8cccc83ed7ce0942576873859b27b33cc4621ec 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -313,7 +313,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { assert(!(tdFileOptions & TD_FILE_EXCL)); fp = fopen(path, mode); if (fp == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } else { @@ -336,14 +336,14 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); #endif if (fd == -1) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } } TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile)); if (pFile == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + // terrno = TSDB_CODE_OUT_OF_MEMORY; if (fd >= 0) close(fd); if (fp != NULL) fclose(fp); return NULL; @@ -464,7 +464,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return 0; + } int64_t nleft = count; int64_t nwritten = 0; @@ -491,6 +493,28 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { return count; } +int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } +#if FILE_WITH_LOCK + taosThreadRwlockWrlock(&(pFile->rwlock)); +#endif + assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); + _lseeki64(pFile->fd, offset, SEEK_SET); + int64_t ret = _write(pFile->fd, buf, count); + _lseeki64(pFile->fd, pos, SEEK_SET); +#else + int64_t ret = pwrite(pFile->fd, buf, count, offset); +#endif +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return ret; +} + int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2f30e8977afbfa6d691fed2cc8ab2c5b62752093..2e68bd5ccd3ad937dbcfce6988fcb7ef6060f31a 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -348,10 +348,11 @@ void taosMemoryTrim(int32_t size) { void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - ASSERT(0); + assert(0); #else #if defined(LINUX) - return memalign(alignment, size); + void* p = memalign(alignment, size); + return p; #else return taosMemoryMalloc(size); #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index e2d8ce95dbdfe7c2f23949df317b9a9e9fa6f99c..f03778de2f8cf9020e9540c581c30f9ca44947b6 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -143,15 +143,17 @@ SConv *gConv = NULL; int32_t convUsed = 0; int32_t gConvMaxNum = 0; -void taosConvInit(void) { +int32_t taosConvInit(void) { gConvMaxNum = 512; gConv = taosMemoryCalloc(gConvMaxNum, sizeof(SConv)); for (int32_t i = 0; i < gConvMaxNum; ++i) { gConv[i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); if ((iconv_t)-1 == gConv[i].conv || (iconv_t)0 == gConv[i].conv) { - ASSERT(0); + return -1; } } + + return 0; } void taosConvDestroy() { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 7ec1da0530087f575f1d7a57fa36fa6c987ef3ee..e1abe8484188106b0a53238881ef55feb1fd2876 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -97,6 +97,7 @@ LONG WINAPI exceptionHandler(LPEXCEPTION_POINTERS exception); #include #include +#include #else @@ -275,34 +276,34 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #endif } + + int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS snprintf(releaseName, maxLen, "Windows"); return 0; #elif defined(_TD_DARWIN_64) - char line[1024]; - size_t size = 0; - int32_t code = -1; + char osversion[32]; + size_t osversion_len = sizeof(osversion) - 1; + int osversion_name[] = { CTL_KERN, KERN_OSRELEASE }; - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; + if (sysctl(osversion_name, 2, osversion, &osversion_len, NULL, 0) == -1) { + return -1; + } - while ((size = taosGetsFile(pFile, sizeof(line), line)) != -1) { - line[size - 1] = '\0'; - if (strncmp(line, "PRETTY_NAME", 11) == 0) { - const char *p = strchr(line, '=') + 1; - if (*p == '"') { - p++; - line[size - 2] = 0; - } - tstrncpy(releaseName, p, maxLen); - code = 0; - break; - } + uint32_t major, minor; + if (sscanf(osversion, "%u.%u", &major, &minor) != 2) { + return -1; + } + if (major >= 20) { + major -= 9; // macOS 11 and newer + sprintf(releaseName, "macOS %u.%u", major, minor); + } else { + major -= 4; // macOS 10.1.1 and newer + sprintf(releaseName, "macOS 10.%d.%d", major, minor); } - taosCloseFile(&pFile); - return code; + return 0; #else char line[1024]; size_t size = 0; @@ -353,6 +354,10 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { code = 0; done |= 1; } + int endPos = strlen(cpuModel)-1; + if (cpuModel[endPos] == '\n') { + cpuModel[endPos] = '\0'; + } taosCloseCmd(&pCmd); pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count"); @@ -484,11 +489,11 @@ int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma) { #ifdef _TD_X86_ // Since the compiler is not support avx/avx2 instructions, the global variables always need to be // set to be false -#if __AVX__ || __AVX2__ - tsSIMDEnable = true; -#else - tsSIMDEnable = false; -#endif +//#if __AVX__ || __AVX2__ +// tsSIMDBuiltins = true; +//#else +// tsSIMDBuiltins = false; +//#endif uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0; @@ -617,14 +622,14 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { return 0; } else { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } #elif defined(_TD_DARWIN_64) struct statvfs info; if (statvfs(dataDir, &info)) { // printf("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { diskSize->total = info.f_blocks * info.f_frsize; @@ -635,7 +640,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { #else struct statvfs info; if (statvfs(dataDir, &info)) { - terrno = TAOS_SYSTEM_ERROR(errno); + // terrno = TAOS_SYSTEM_ERROR(errno); return -1; } else { diskSize->total = info.f_blocks * info.f_frsize; diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 68dfba14e95a0bb34af7c60793f6af8bda234a7f..cd4324a5928e11497d3f98276a8cff800e369395 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -572,7 +572,7 @@ int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { offsetInitFinished = true; } else { while (!offsetInitFinished) - ; // Ensure initialization is completed. + ; // Ensure initialization is completed. } GetSystemTimeAsFileTime(&f); diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 4d6875d26376cad79b2361a772cfd6f6f5f1829a..057d3a620c714b22e0dea9643f672782c93424ed 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "talgo.h" +#include "tlog.h" #define doswap(__left, __right, __size, __buf) \ do { \ diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index d84a3d25c604d0089e1abc5ca2778a255a13c859..3edd067ce2aafa35e96cf632badf04f822ce2454 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1120,7 +1120,7 @@ int32_t WCSPatternMatch(const TdUcs4 *patterStr, const TdUcs4 *str, size_t size, return TSDB_PATTERN_NOMATCH; } - return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; + return (j >= size || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 586586f74238c49bd4e5352666c5a37b1d39d449..609a386367d825bca096b12857b8e8ece013c49d 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -293,9 +293,9 @@ static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, } SDiskCfg cfg = {0}; - tstrncpy(cfg.dir, value, sizeof(cfg.dir)); - cfg.level = atoi(level); - cfg.primary = atoi(primary); + tstrncpy(cfg.dir, pItem->str, sizeof(cfg.dir)); + cfg.level = level ? atoi(level) : 0; + cfg.primary = primary ? atoi(primary) : 1; void *ret = taosArrayPush(pItem->array, &cfg); if (ret == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -660,12 +660,12 @@ int32_t cfgLoadFromEnvVar(SConfig *pConfig) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -703,12 +703,12 @@ int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -768,12 +768,12 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -828,12 +828,12 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } @@ -895,7 +895,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { // code = cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; -// if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { +// if (strcasecmp(name, "dataDir") == 0) { // code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); // if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; // } @@ -993,12 +993,13 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { paGetToken(value2 + vlen2 + 1, &value3, &vlen3); if (vlen3 != 0) value3[vlen3] = 0; } - if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + + code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); + if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; + + if (strcasecmp(name, "dataDir") == 0) { code = cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; - } else { - code = cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); - if (code != 0 && terrno != TSDB_CODE_CFG_NOT_FOUND) break; } } } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f5d11e3e960c802b92123abb8d507b85c8348342..ba00d68320342941fae1ef778598d58e0d889253 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -46,8 +46,6 @@ STaosError errors[] = { #endif // rpc -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failure") -TAOS_DEFINE_ERROR(TSDB_CODE_RPC_REDIRECT, "Database not ready, need retry") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") @@ -56,12 +54,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_TIMEOUT, "Conn read timeout") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_TIME_UNSYNCED, "Client and server's time is not synchronized") -TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_OPS_NOT_SUPPORT, "Operation not supported") -TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_MEMORY, "Out of Memory") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") TAOS_DEFINE_ERROR(TSDB_CODE_REF_INVALID_ID, "Invalid Ref ID") @@ -71,8 +66,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REF_NOT_EXIST, "Ref is not there") TAOS_DEFINE_ERROR(TSDB_CODE_APP_ERROR, "Unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RANGE, "Out of range") -TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Shared memory") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SHM_ID, "Invalid SHM ID") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG_LEN, "Invalid message len") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") @@ -115,11 +108,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_PASS_LENGTH, "Invalid password") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_DB_LENGTH, "Database name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH, "Table name too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_CONNECTION, "Invalid connection") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_OUT_OF_MEMORY, "System out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CACHE_ERASED, "Query cache erased") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_CANCELLED, "Query terminated") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SORTED_RES_TOO_MANY, "Result set too large to be sorted") // too many result for ordered super table projection query -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, "Application error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, "No write permission") @@ -145,34 +136,23 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, "Invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, "Invalid connection id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_IS_RUNNING, "mnode is alreay running") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CONFIG_SYNC, "failed to config sync") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_START_SYNC, "failed to start sync") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_CREATE_DIR, "failed to create mnode dir") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_FAILED_TO_INIT_STEP, "failed to init components") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_DISABLED, "User is disabled") // mnode-sdb TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_ALREADY_THERE, "Object already there") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_APP_ERROR, "Unexpected generic error in sdb") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_NOT_THERE, "Object not there") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_KEY_TYPE, "Invalid key type") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_ACTION_TYPE, "Invalid action type") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_STATUS_TYPE, "Invalid status type") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_VER, "Invalid raw data version") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_LEN, "Invalid raw data len") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_DATA_CONTENT, "Invalid raw data content") -TAOS_DEFINE_ERROR(TSDB_CODE_SDB_INVALID_WAl_VER, "Invalid wal version") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_CREATING, "Object is creating") TAOS_DEFINE_ERROR(TSDB_CODE_SDB_OBJ_DROPPING, "Object is dropping") @@ -277,6 +257,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CONFLICT, "Conflict transaction not completed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog is null") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction") // mnode-mq @@ -301,6 +282,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB, "Stream temporarily does not support source db having replica > 1") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") @@ -308,20 +290,30 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "SMA does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma option") // dnode -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_OFFLINE, "Node is offline") -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_ALREADY_DEPLOYED, "Node already deployed") -TAOS_DEFINE_ERROR(TSDB_CODE_NODE_NOT_DEPLOYED, "Node not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_FOUND, "Mnode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_ALREADY_DEPLOYED, "Mnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_MNODE_NOT_DEPLOYED, "Mnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_NOT_FOUND, "Qnode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_ALREADY_DEPLOYED, "Qnode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_QNODE_NOT_DEPLOYED, "Qnode not deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_FOUND, "Snode not found") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deployed") +TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Invalid Vgroup ID") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Hash value mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_HASH_MISMATCH, "Vnode hash mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_NOT_EXISTS, "Table column not exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subscribed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -331,8 +323,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists" TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISK_PERMISSIONS, "No permission for disk files") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, "Data file(s) corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, "Tag too old") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, "Timestamp data out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, "Submit message is messed up") @@ -356,8 +346,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_IN_OTHER_STABLE, "Table already exists // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, "Invalid handle") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_MSG, "Invalid message") // failed to validate the sql expression msg by vnode -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_OUT_OF_MEMORY, "System out of memory") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_APP_ERROR, "Unexpected generic error in query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, "Duplicated join key") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, "Tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, "Query not ready") @@ -421,8 +409,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INVALID_CONFIG, "TQ invalid config") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_INIT_FAILED, "TQ init falied") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_DISK_PERMISSIONS, "TQ no disk permissions") -TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_CORRUPTED, "TQ file corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_TQ_OUT_OF_MEMORY, "TQ out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FILE_ALREADY_EXISTS, "TQ file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_FAILED_TO_CREATE_DIR, "TQ failed to create dir") TAOS_DEFINE_ERROR(TSDB_CODE_TQ_META_NO_SUCH_KEY, "TQ meta no such key") @@ -433,11 +419,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND, "TQ table schema not f TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no commited offset") // wal -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "WAL unexpected generic error") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") @@ -451,7 +435,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_MOUNT_AT_TIER, "tfs no mount at tier" TAOS_DEFINE_ERROR(TSDB_CODE_FS_FILE_ALREADY_EXISTS, "tfs file already exists") TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") -TAOS_DEFINE_ERROR(TSDB_CODE_FS_APP_ERROR, "tfs out of memory") // catalog TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog internal error") @@ -603,7 +586,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in ca TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo creation error") -TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FILE_CORRUPTED, "Rsma file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") diff --git a/source/util/src/thash.c b/source/util/src/thash.c index a0411483cae13e2637078b59c36599ebbd9625a6..e9548613aa9b3ce2edc4f00db8b1662202a03b38 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -244,7 +244,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp capacity = 4; } - SHashObj *pHashObj = (SHashObj *)taosMemoryCalloc(1, sizeof(SHashObj)); + SHashObj *pHashObj = (SHashObj *)taosMemoryMalloc(sizeof(SHashObj)); if (pHashObj == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -264,7 +264,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)taosMemoryCalloc(pHashObj->capacity, sizeof(void *)); + pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *)); if (pHashObj->hashList == NULL) { taosMemoryFree(pHashObj); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -279,7 +279,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return NULL; } - void *p = taosMemoryCalloc(pHashObj->capacity, sizeof(SHashEntry)); + void *p = taosMemoryMalloc(pHashObj->capacity * sizeof(SHashEntry)); if (p == NULL) { taosArrayDestroy(pHashObj->pMemBlock); taosMemoryFree(pHashObj->hashList); @@ -290,6 +290,9 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp for (int32_t i = 0; i < pHashObj->capacity; ++i) { pHashObj->hashList[i] = (void *)((char *)p + i * sizeof(SHashEntry)); + pHashObj->hashList[i]->num = 0; + pHashObj->hashList[i]->latch = 0; + pHashObj->hashList[i]->next = NULL; } taosArrayPush(pHashObj->pMemBlock, &p); diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index cc823decf4754617cd98d33e2fe1b6f85dc52013..48638af8d59e2fa326db88e3c10c7fa85be8e741 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -181,7 +181,7 @@ int32_t tjsonGetObjectValueString(const SJson* pJson, char** pValueString) { int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } strcpy(pVal, p); return TSDB_CODE_SUCCESS; @@ -190,7 +190,7 @@ int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } *pVal = strdup(p); return TSDB_CODE_SUCCESS; @@ -199,7 +199,7 @@ int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } #ifdef WINDOWS sscanf(p, "%" PRId64, pVal); @@ -233,7 +233,7 @@ int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } #ifdef WINDOWS sscanf(p, "%" PRIu64, pVal); @@ -259,6 +259,9 @@ int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pV int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal) { const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (NULL == pObject) { + return TSDB_CODE_SUCCESS; + } if (!cJSON_IsBool(pObject)) { return TSDB_CODE_FAILED; } @@ -268,6 +271,9 @@ int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal) { int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal) { const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (NULL == pObject) { + return TSDB_CODE_SUCCESS; + } if (!cJSON_IsNumber(pObject)) { return TSDB_CODE_FAILED; } @@ -282,7 +288,7 @@ SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index) { return cJSON_GetAr int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj) { SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); if (NULL == pJsonObj) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } return func(pJsonObj, pObj); } @@ -294,7 +300,7 @@ int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, v SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); if (NULL == pJsonObj) { - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } *pObj = taosMemoryCalloc(1, objSize); if (NULL == *pObj) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index fc9d90c9857ac1be20722ec22134dbb3da94b3b8..f6f814d82b8f1de69e28790bb6407c629446590e 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -72,6 +72,7 @@ static int32_t tsDaylightActive; /* Currently in daylight saving time. */ bool tsLogEmbedded = 0; bool tsAsyncLog = true; +bool tsAssert = true; int32_t tsNumOfLogLines = 10000000; int32_t tsLogKeepDays = 0; LogFp tsLogFp = NULL; @@ -444,6 +445,9 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { taosUpdateLogNums(level); +#if 0 + // DEBUG_FATAL and DEBUG_ERROR are duplicated + // fsync will cause thread blocking and may also generate log misalignment in case of asyncLog if (tsAsyncLog && level != DEBUG_FATAL) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { @@ -452,6 +456,13 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b taosFsyncFile(tsLogObj.logHandle->pFile); } } +#else + if (tsAsyncLog) { + taosPushLogBuffer(tsLogObj.logHandle, buffer, len); + } else { + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); + } +#endif if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); @@ -496,7 +507,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons if (!osLogSpaceAvailable()) return; if (!(dflag & DEBUG_FILE) && !(dflag & DEBUG_SCREEN)) return; - char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); + char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); int32_t len = taosBuildLogHead(buffer, flags); va_list argpointer; @@ -778,3 +789,37 @@ cmp_end: return ret; } + +bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...) { + if (condition) return false; + + const char *flags = "UTL FATAL "; + ELogLevel level = DEBUG_FATAL; + int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag + char buffer[LOG_MAX_LINE_BUFFER_SIZE]; + int32_t len = taosBuildLogHead(buffer, flags); + + va_list argpointer; + va_start(argpointer, format); + len = len + vsnprintf(buffer + len, LOG_MAX_LINE_BUFFER_SIZE - len, format, argpointer); + va_end(argpointer); + buffer[len++] = '\n'; + buffer[len] = 0; + taosPrintLogImp(1, 255, buffer, len); + + taosPrintLog(flags, level, dflag, "tAssert at file %s:%d exit:%d", file, line, tsAssert); + taosPrintTrace(flags, level, dflag); + + if (tsAssert) { + // taosCloseLog(); + taosMsleep(300); + +#ifdef NDEBUG + abort(); +#else + assert(0); +#endif + } + + return true; +} \ No newline at end of file diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index e1a43ace47adf998fbd872557f8e6a618dd4f718..37874ae250264dfe69a1669878fbe33ab9683e6d 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -30,6 +30,7 @@ struct SDiskbasedBuf { TdFilePtr pFile; int32_t allocateId; // allocated page id char* path; // file path + char* prefix; // file name prefix int32_t pageSize; // current used page size int32_t inMemPages; // numOfPages that are allocated in memory SList* freePgList; // free page list @@ -48,6 +49,12 @@ struct SDiskbasedBuf { }; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { + if (pBuf->path == NULL) { // prepare the file name when needed it + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); + pBuf->path = taosMemoryStrDup(path); + } + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (pBuf->pFile == NULL) { @@ -356,10 +363,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->assistBuf = taosMemoryMalloc(pPBuf->pageSize + 2); // EXTRA BYTES pPBuf->all = taosHashInit(10, fn, true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "paged-buf", path); - pPBuf->path = strdup(path); + pPBuf->prefix = (char*) dir; pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); @@ -507,7 +511,9 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { dBufPrintStatis(pBuf); + bool needRemoveFile = false; if (pBuf->pFile != NULL) { + needRemoveFile = true; uDebug( "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " "size:%.2f Kb, %s\n", @@ -534,9 +540,13 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { } } - if (taosRemoveFile(pBuf->path) < 0) { - uDebug("WARNING tPage remove file failed. path=%s", pBuf->path); + if (needRemoveFile) { + int32_t ret = taosRemoveFile(pBuf->path); + if (ret != 0) { // print the error and discard this error info + uDebug("WARNING tPage remove file failed. path=%s, code:%s", pBuf->path, strerror(errno)); + } } + taosMemoryFreeClear(pBuf->path); size_t n = taosArrayGetSize(pBuf->pIdList); diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 0f992184b579fa8ab1e4855f43e61f81b772c07c..42b63588930712d2e132fd7e8bf860aef3622ff9 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -109,20 +109,24 @@ int64_t taosQueueMemorySize(STaosQueue *queue) { return memOfItems; } -void *taosAllocateQitem(int32_t size, EQItype itype) { +void *taosAllocateQitem(int32_t size, EQItype itype, int64_t dataSize) { STaosQnode *pNode = taosMemoryCalloc(1, sizeof(STaosQnode) + size); if (pNode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pNode->dataSize = dataSize; pNode->size = size; pNode->itype = itype; pNode->timestamp = taosGetTimestampUs(); if (itype == RPC_QITEM) { - int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size); + int64_t alloced = atomic_add_fetch_64(&tsRpcQueueMemoryUsed, size + dataSize); if (alloced > tsRpcQueueMemoryAllowed) { + uError("failed to alloc qitem, size:%" PRId64 " alloc:%" PRId64 " allowed:%" PRId64, size + dataSize, alloced, + tsRpcQueueMemoryUsed); + atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, size + dataSize); taosMemoryFree(pNode); terrno = TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE; return NULL; @@ -139,8 +143,8 @@ void taosFreeQitem(void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); - if (pNode->itype > 0) { - int64_t alloced = atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, pNode->size); + if (pNode->itype == RPC_QITEM) { + int64_t alloced = atomic_sub_fetch_64(&tsRpcQueueMemoryUsed, pNode->size + pNode->dataSize); uTrace("item:%p, node:%p is freed, alloc:%" PRId64, pItem, pNode, alloced); } else { uTrace("item:%p, node:%p is freed", pItem, pNode); diff --git a/source/util/src/tref.c b/source/util/src/tref.c index b4322464e2107b71a19acf5a36dd7ee9a51278df..75a16b0ad706668edc1ad2567aa9393a06fc685d 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -67,14 +67,14 @@ int32_t taosOpenRef(int32_t max, RefFp fp) { nodeList = taosMemoryCalloc(sizeof(SRefNode *), (size_t)max); if (nodeList == NULL) { - terrno = TSDB_CODE_REF_NO_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } lockedBy = taosMemoryCalloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { taosMemoryFree(nodeList); - terrno = TSDB_CODE_REF_NO_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -164,7 +164,7 @@ int64_t taosAddRef(int32_t rsetId, void *p) { pNode = taosMemoryCalloc(sizeof(SRefNode), 1); if (pNode == NULL) { - terrno = TSDB_CODE_REF_NO_MEMORY; + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } diff --git a/source/util/test/hashTest.cpp b/source/util/test/hashTest.cpp index 76fb9c70679c30e537a9ece9b1202b2c9b5da08e..00379a755bde167f852fdb5283fcac58b227eb35 100644 --- a/source/util/test/hashTest.cpp +++ b/source/util/test/hashTest.cpp @@ -6,6 +6,7 @@ #include "taos.h" #include "taosdef.h" #include "thash.h" +#include "tlog.h" namespace { diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py index 163cdd0055640d2ff4261096801ed2813747a42c..ba296584ba6e81e2f8db6860240c6535cfc62cbf 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/commandline.py @@ -56,7 +56,7 @@ class TDTestCase: def run(self): binPath = self.getPath() - cmd = "%s -F 7 -H 9 -n 10 -t 2 -x -y -M -C -d newtest -l 5 -A binary,nchar\(31\) -b tinyint,binary\(23\),bool,nchar -w 29 -E -m $%%^*" %binPath + cmd = "%s -F 7 -n 10 -t 2 -x -y -M -C -d newtest -l 5 -A binary,nchar\(31\) -b tinyint,binary\(23\),bool,nchar -w 29 -E -m $%%^*" %binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("use newtest") diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 6be5117b08e8976c11e86b8cbb8072a3e3f8f876..25a8d92f7e9b45e55011d5fe0bb241bb68962340 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -28,7 +28,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py index 73894d5e333704e10e6190a417396ee5eefd24a8..d706eea068221602e057baa236b1dd671ddd1674 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/invalid_commandline.py @@ -25,7 +25,7 @@ class TDTestCase: return def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) + # comment off by Shuduo for CI self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -53,7 +53,7 @@ class TDTestCase: def run(self): binPath = self.getPath() - cmd = "%s -F abc -P abc -I abc -T abc -H abc -i abc -S abc -B abc -r abc -t abc -n abc -l abc -w abc -w 16385 -R abc -O abc -a abc -n 2 -t 2 -r 1 -y" %binPath + cmd = "%s -F abc -P abc -I abc -T abc -i abc -S abc -B abc -r abc -t abc -n abc -l abc -w abc -w 16385 -R abc -O abc -a abc -n 2 -t 2 -r 1 -y" %binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.query("select count(*) from test.meters") diff --git a/tests/docs-examples-test/csharp.sh b/tests/docs-examples-test/csharp.sh index 21c19b9b3d7dae0680381b112e84f44a2023f5dd..c08ffd6d62055fc580594944ad70ffa974b34561 100644 --- a/tests/docs-examples-test/csharp.sh +++ b/tests/docs-examples-test/csharp.sh @@ -28,10 +28,10 @@ taos -s "drop database if exists test" dotnet run --project optsJSON/optsJSON.csproj taos -s "create database if not exists test" -# dotnet run --project wsConnect/wsConnect.csproj -# dotnet run --project wsInsert/wsInsert.csproj -# dotnet run --project wsStmt/wsStmt.csproj -# dotnet run --project wsQuery/wsQuery.csproj +dotnet run --project wsConnect/wsConnect.csproj +dotnet run --project wsInsert/wsInsert.csproj +dotnet run --project wsStmt/wsStmt.csproj +dotnet run --project wsQuery/wsQuery.csproj taos -s "drop database if exists test" -taos -s "drop database if exists power" \ No newline at end of file +taos -s "drop database if exists power" diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6c5c1718b2cc906ee3b46cd836a6e1211fd3a9d3..5361ba356fccc9a31f96e35b6587c3a777661776 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,6 +10,7 @@ ,,y,script,./test.sh -f tsim/user/password.sim ,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,y,script,./test.sh -f tsim/user/privilege_topic.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim ,,y,script,./test.sh -f tsim/db/alter_replica_13.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim @@ -113,7 +114,6 @@ ,,y,script,./test.sh -f tsim/parser/first_last.sim ,,y,script,./test.sh -f tsim/parser/fill_stb.sim ,,y,script,./test.sh -f tsim/parser/interp.sim -#,,y,script,./test.sh -f tsim/parser/limit2.sim ,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim ,,y,script,./test.sh -f tsim/parser/function.sim ,,y,script,./test.sh -f tsim/parser/groupby-basic.sim @@ -174,6 +174,7 @@ ,,y,script,./test.sh -f tsim/query/scalarNull.sim ,,y,script,./test.sh -f tsim/query/session.sim ,,y,script,./test.sh -f tsim/query/udf.sim +,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim ,,y,script,./test.sh -f tsim/mnode/basic1.sim @@ -624,12 +625,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -,,,system-test,python3 ./test.py -f 2-query/stablity.py -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py @@ -789,12 +790,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2 @@ -882,12 +883,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 @@ -975,10 +976,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 @@ -1038,4 +1039,4 @@ ,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh -,,n,docs-examples-test,bash go.sh +#,,n,docs-examples-test,bash go.sh diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 4140728dcdaaf94f79273781c8d68fb35e78acf8..3c39b05e699b0f90898eed9363f7e3770f8c51ce 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2022,11 +2022,10 @@ class TdSuperTable: conf.set("group.id", "tg2") conf.set("td.connect.user", "root") conf.set("td.connect.pass", "taosdata") - conf.set("enable.auto. - ", "true") - def tmq_commit_cb_print(tmq, resp, offset, param=None): - print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - conf.set_auto_commit_cb(tmq_commit_cb_print, None) +# conf.set("enable.auto.commit", "true") +# def tmq_commit_cb_print(tmq, resp, offset, param=None): +# print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") +# conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: diff --git a/examples/c/api_with_reqid_test.c b/tests/script/api/api_with_reqid_test.c similarity index 100% rename from examples/c/api_with_reqid_test.c rename to tests/script/api/api_with_reqid_test.c diff --git a/tests/script/api/apitest.c b/tests/script/api/apitest.c new file mode 100644 index 0000000000000000000000000000000000000000..7280522021b99a75f8315d9dc730eb5dc1eb7176 --- /dev/null +++ b/tests/script/api/apitest.c @@ -0,0 +1,936 @@ +// sample code to verify all TDengine API +// to compile: gcc -o apitest apitest.c -ltaos + +#include "cJSON.h" +#include "taoserror.h" + +#include +#include +#include +#include +#include "../../include/client/taos.h" + +static int64_t count = 10000; + +int64_t genReqid() { + count += 100; + return count; +} + +static void prepare_data(TAOS* taos) { + TAOS_RES* result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test precision 'us';"); + taos_free_result(result); + usleep(100000); + taos_select_db(taos, "test"); + + result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + taos_free_result(result); + + result = taos_query(taos, "create table t0 using meters tags(0);"); + taos_free_result(result); + result = taos_query(taos, "create table t1 using meters tags(1);"); + taos_free_result(result); + result = taos_query(taos, "create table t2 using meters tags(2);"); + taos_free_result(result); + result = taos_query(taos, "create table t3 using meters tags(3);"); + taos_free_result(result); + result = taos_query(taos, "create table t4 using meters tags(4);"); + taos_free_result(result); + result = taos_query(taos, "create table t5 using meters tags(5);"); + taos_free_result(result); + result = taos_query(taos, "create table t6 using meters tags(6);"); + taos_free_result(result); + result = taos_query(taos, "create table t7 using meters tags(7);"); + taos_free_result(result); + result = taos_query(taos, "create table t8 using meters tags(8);"); + taos_free_result(result); + result = taos_query(taos, "create table t9 using meters tags(9);"); + taos_free_result(result); + + result = taos_query(taos, + "insert into t0 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " t1 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:02:00.000', 0)" + " ('2020-01-01 00:03:00.000', 0)" + " t2 values('2020-01-01 00:00:00.000', 0)" + " ('2020-01-01 00:01:00.000', 0)" + " ('2020-01-01 00:01:01.000', 0)" + " ('2020-01-01 00:01:02.000', 0)" + " t3 values('2020-01-01 00:01:02.000', 0)" + " t4 values('2020-01-01 00:01:02.000', 0)" + " t5 values('2020-01-01 00:01:02.000', 0)" + " t6 values('2020-01-01 00:01:02.000', 0)" + " t7 values('2020-01-01 00:01:02.000', 0)" + " t8 values('2020-01-01 00:01:02.000', 0)" + " t9 values('2020-01-01 00:01:02.000', 0)"); + int affected = taos_affected_rows(result); + if (affected != 18) { + printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); + } + taos_free_result(result); + // super tables subscription + usleep(1000000); +} + +static int print_result(TAOS_RES* res, int blockFetch) { + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + int nRows = 0; + + if (blockFetch) { + int rows = 0; + while ((rows = taos_fetch_block(res, &row))) { + // for (int i = 0; i < rows; i++) { + // char temp[256]; + // taos_print_row(temp, row + i, fields, num_fields); + // puts(temp); + // } + nRows += rows; + } + } else { + while ((row = taos_fetch_row(res))) { + char temp[256] = {0}; + taos_print_row(temp, row, fields, num_fields); + puts(temp); + nRows++; + } + } + + printf("%d rows consumed.\n", nRows); + return nRows; +} + +static void check_row_count(int line, TAOS_RES* res, int expected) { + int actual = print_result(res, expected % 2); + if (actual != expected) { + printf("\033[31mline %d: row count mismatch, expected: %d, actual: %d\033[0m\n", line, expected, actual); + } else { + printf("line %d: %d rows consumed as expected\n", line, actual); + } +} + +static void verify_query(TAOS* taos) { + prepare_data(taos); + + int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9"); + if (code != 0) { + printf("\033[31mfailed to load table info: 0x%08x\033[0m\n", code); + } + + code = taos_validate_sql(taos, "select * from nonexisttable"); + if (code == 0) { + printf("\033[31mimpossible, the table does not exists\033[0m\n"); + } + + code = taos_validate_sql(taos, "select * from meters"); + if (code != 0) { + printf("\033[31mimpossible, the table does exists: 0x%08x\033[0m\n", code); + } + + TAOS_RES* res = taos_query_with_reqid(taos, "select * from meters", genReqid()); + check_row_count(__LINE__, res, 18); + printf("result precision is: %d\n", taos_result_precision(res)); + int c = taos_field_count(res); + printf("field count is: %d\n", c); + int* lengths = taos_fetch_lengths(res); + for (int i = 0; i < c; i++) { + printf("length of column %d is %d\n", i, lengths[i]); + } + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from t0", genReqid()); + check_row_count(__LINE__, res, 3); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from nonexisttable", genReqid()); + code = taos_errno(res); + printf("code=%d, error msg=%s\n", code, taos_errstr(res)); + taos_free_result(res); + + res = taos_query_with_reqid(taos, "select * from meters", genReqid()); + taos_stop_query(res); + taos_free_result(res); +} + +void subscribe_callback(TAOS_SUB* tsub, TAOS_RES* res, void* param, int code) { + int rows = print_result(res, *(int*)param); + printf("%d rows consumed in subscribe_callback\n", rows); +} + +void verify_prepare(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + + usleep(100000); + result = taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = + "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " + "binary(40), blob nchar(10))"; + result = taos_query_with_reqid(taos, sql, genReqid()); + code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + // insert 10 records + struct { + int64_t ts; + int8_t b; + int8_t v1; + int16_t v2; + int32_t v4; + int64_t v8; + float f4; + double f8; + char bin[40]; + char blob[80]; + } v = {0}; + + int32_t boolLen = sizeof(int8_t); + int32_t sintLen = sizeof(int16_t); + int32_t intLen = sizeof(int32_t); + int32_t bintLen = sizeof(int64_t); + int32_t floatLen = sizeof(float); + int32_t doubleLen = sizeof(double); + int32_t binLen = sizeof(v.bin); + int32_t ncharLen = 30; + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_MULTI_BIND params[10]; + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts); + params[0].buffer = &v.ts; + params[0].length = &bintLen; + params[0].is_null = NULL; + params[0].num = 1; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b); + params[1].buffer = &v.b; + params[1].length = &boolLen; + params[1].is_null = NULL; + params[1].num = 1; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1); + params[2].buffer = &v.v1; + params[2].length = &boolLen; + params[2].is_null = NULL; + params[2].num = 1; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2); + params[3].buffer = &v.v2; + params[3].length = &sintLen; + params[3].is_null = NULL; + params[3].num = 1; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4); + params[4].buffer = &v.v4; + params[4].length = &intLen; + params[4].is_null = NULL; + params[4].num = 1; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8); + params[5].buffer = &v.v8; + params[5].length = &bintLen; + params[5].is_null = NULL; + params[5].num = 1; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4); + params[6].buffer = &v.f4; + params[6].length = &floatLen; + params[6].is_null = NULL; + params[6].num = 1; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8); + params[7].buffer = &v.f8; + params[7].length = &doubleLen; + params[7].is_null = NULL; + params[7].num = 1; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin); + params[8].buffer = v.bin; + params[8].length = &binLen; + params[8].is_null = NULL; + params[8].num = 1; + + strcpy(v.blob, "一二三四五六七八九十"); + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob); + params[9].buffer = v.blob; + params[9].length = &ncharLen; + params[9].is_null = NULL; + params[9].num = 1; + + char is_null = 1; + + sql = "insert into m1 values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0) { + printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + v.ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts += 1; + for (int j = 1; j < 10; ++j) { + params[j].is_null = ((i == j) ? &is_null : 0); + } + v.b = (int8_t)i % 2; + v.v1 = (int8_t)i; + v.v2 = (int16_t)(i * 2); + v.v4 = (int32_t)(i * 4); + v.v8 = (int64_t)(i * 8); + v.f4 = (float)(i * 40); + v.f8 = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin); ++j) { + v.bin[j] = (char)(i + '0'); + } + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + } + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + v.v1 = 5; + v.v2 = 15; + taos_stmt_bind_param(stmt, params + 2); + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD* fields = taos_fetch_fields(result); + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + char temp[256] = {0}; + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); +} + +void verify_prepare2(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = + "create table m1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " + "binary(40), blob nchar(10))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t* t8_len = malloc(sizeof(int32_t) * 10); + int32_t* t16_len = malloc(sizeof(int32_t) * 10); + int32_t* t32_len = malloc(sizeof(int32_t) * 10); + int32_t* t64_len = malloc(sizeof(int32_t) * 10); + int32_t* float_len = malloc(sizeof(int32_t) * 10); + int32_t* double_len = malloc(sizeof(int32_t) * 10); + int32_t* bin_len = malloc(sizeof(int32_t) * 10); + int32_t* blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + sql = "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0) { + printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + code = taos_stmt_set_tbname(stmt, "m1"); + if (code != 0) { + printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + int64_t ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts++; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]); ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + taos_stmt_bind_param_batch(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + TAOS_MULTI_BIND qparams[2]; + + int8_t v1 = 5; + int16_t v2 = 15; + int32_t tinyLen = sizeof(v1); + int32_t smallLen = sizeof(v2); + + qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; + qparams[0].buffer_length = sizeof(v1); + qparams[0].buffer = &v1; + qparams[0].length = &tinyLen; + qparams[0].is_null = NULL; + qparams[0].num = 1; + + qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + qparams[1].buffer_length = sizeof(v2); + qparams[1].buffer = &v2; + qparams[1].length = &smallLen; + qparams[1].is_null = NULL; + qparams[1].num = 1; + + taos_stmt_bind_param(stmt, qparams); + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD* fields = taos_fetch_fields(result); + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + char temp[256] = {0}; + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); +} + +void verify_prepare3(TAOS* taos) { + TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test;"); + + int code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create database, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + usleep(100000); + taos_select_db(taos, "test"); + + // create table + const char* sql = + "create stable st1 (ts timestamp, b bool, v1 tinyint, v2 smallint, v4 int, v8 bigint, f4 float, f8 double, bin " + "binary(40), blob nchar(10)) tags (id1 int, id2 binary(40))"; + result = taos_query(taos, sql); + code = taos_errno(result); + if (code != 0) { + printf("\033[31mfailed to create table, reason:%s\033[0m\n", taos_errstr(result)); + taos_free_result(result); + return; + } + taos_free_result(result); + + TAOS_MULTI_BIND tags[2]; + + int32_t id1 = 1; + char id2[40] = "abcdefghijklmnopqrstuvwxyz0123456789"; + int32_t id2_len = (int32_t)strlen(id2); + + tags[0].buffer_type = TSDB_DATA_TYPE_INT; + tags[0].buffer_length = sizeof(int); + tags[0].buffer = &id1; + tags[0].length = NULL; + tags[0].is_null = NULL; + tags[0].num = 1; + + tags[1].buffer_type = TSDB_DATA_TYPE_BINARY; + tags[1].buffer_length = sizeof(id2); + tags[1].buffer = id2; + tags[1].length = &id2_len; + tags[1].is_null = NULL; + tags[1].num = 1; + + // insert 10 records + struct { + int64_t ts[10]; + int8_t b[10]; + int8_t v1[10]; + int16_t v2[10]; + int32_t v4[10]; + int64_t v8[10]; + float f4[10]; + double f8[10]; + char bin[10][40]; + char blob[10][80]; + } v; + + int32_t* t8_len = malloc(sizeof(int32_t) * 10); + int32_t* t16_len = malloc(sizeof(int32_t) * 10); + int32_t* t32_len = malloc(sizeof(int32_t) * 10); + int32_t* t64_len = malloc(sizeof(int32_t) * 10); + int32_t* float_len = malloc(sizeof(int32_t) * 10); + int32_t* double_len = malloc(sizeof(int32_t) * 10); + int32_t* bin_len = malloc(sizeof(int32_t) * 10); + int32_t* blob_len = malloc(sizeof(int32_t) * 10); + + TAOS_STMT* stmt = taos_stmt_init(taos); + TAOS_MULTI_BIND params[10]; + char is_null[10] = {0}; + + params[0].buffer_type = TSDB_DATA_TYPE_TIMESTAMP; + params[0].buffer_length = sizeof(v.ts[0]); + params[0].buffer = v.ts; + params[0].length = t64_len; + params[0].is_null = is_null; + params[0].num = 10; + + params[1].buffer_type = TSDB_DATA_TYPE_BOOL; + params[1].buffer_length = sizeof(v.b[0]); + params[1].buffer = v.b; + params[1].length = t8_len; + params[1].is_null = is_null; + params[1].num = 10; + + params[2].buffer_type = TSDB_DATA_TYPE_TINYINT; + params[2].buffer_length = sizeof(v.v1[0]); + params[2].buffer = v.v1; + params[2].length = t8_len; + params[2].is_null = is_null; + params[2].num = 10; + + params[3].buffer_type = TSDB_DATA_TYPE_SMALLINT; + params[3].buffer_length = sizeof(v.v2[0]); + params[3].buffer = v.v2; + params[3].length = t16_len; + params[3].is_null = is_null; + params[3].num = 10; + + params[4].buffer_type = TSDB_DATA_TYPE_INT; + params[4].buffer_length = sizeof(v.v4[0]); + params[4].buffer = v.v4; + params[4].length = t32_len; + params[4].is_null = is_null; + params[4].num = 10; + + params[5].buffer_type = TSDB_DATA_TYPE_BIGINT; + params[5].buffer_length = sizeof(v.v8[0]); + params[5].buffer = v.v8; + params[5].length = t64_len; + params[5].is_null = is_null; + params[5].num = 10; + + params[6].buffer_type = TSDB_DATA_TYPE_FLOAT; + params[6].buffer_length = sizeof(v.f4[0]); + params[6].buffer = v.f4; + params[6].length = float_len; + params[6].is_null = is_null; + params[6].num = 10; + + params[7].buffer_type = TSDB_DATA_TYPE_DOUBLE; + params[7].buffer_length = sizeof(v.f8[0]); + params[7].buffer = v.f8; + params[7].length = double_len; + params[7].is_null = is_null; + params[7].num = 10; + + params[8].buffer_type = TSDB_DATA_TYPE_BINARY; + params[8].buffer_length = sizeof(v.bin[0]); + params[8].buffer = v.bin; + params[8].length = bin_len; + params[8].is_null = is_null; + params[8].num = 10; + + params[9].buffer_type = TSDB_DATA_TYPE_NCHAR; + params[9].buffer_length = sizeof(v.blob[0]); + params[9].buffer = v.blob; + params[9].length = blob_len; + params[9].is_null = is_null; + params[9].num = 10; + + sql = "insert into ? using st1 tags(?,?) values(?,?,?,?,?,?,?,?,?,?)"; + code = taos_stmt_prepare(stmt, sql, 0); + if (code != 0) { + printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + code = taos_stmt_set_tbname_tags(stmt, "m1", tags); + if (code != 0) { + printf("\033[31mfailed to execute taos_stmt_prepare. error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + int64_t ts = 1591060628000; + for (int i = 0; i < 10; ++i) { + v.ts[i] = ts++; + is_null[i] = 0; + + v.b[i] = (int8_t)i % 2; + v.v1[i] = (int8_t)i; + v.v2[i] = (int16_t)(i * 2); + v.v4[i] = (int32_t)(i * 4); + v.v8[i] = (int64_t)(i * 8); + v.f4[i] = (float)(i * 40); + v.f8[i] = (double)(i * 80); + for (int j = 0; j < sizeof(v.bin[0]); ++j) { + v.bin[i][j] = (char)(i + '0'); + } + strcpy(v.blob[i], "一二三四五六七八九十"); + + t8_len[i] = sizeof(int8_t); + t16_len[i] = sizeof(int16_t); + t32_len[i] = sizeof(int32_t); + t64_len[i] = sizeof(int64_t); + float_len[i] = sizeof(float); + double_len[i] = sizeof(double); + bin_len[i] = sizeof(v.bin[0]); + blob_len[i] = (int32_t)strlen(v.blob[i]); + } + + taos_stmt_bind_param_batch(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute insert statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + taos_stmt_close(stmt); + + // query the records + stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); + + TAOS_MULTI_BIND qparams[2]; + + int8_t v1 = 5; + int16_t v2 = 15; + int32_t tinyLen = sizeof(v1); + int32_t smallLen = sizeof(v2); + + qparams[0].buffer_type = TSDB_DATA_TYPE_TINYINT; + qparams[0].buffer_length = sizeof(v1); + qparams[0].buffer = &v1; + qparams[0].length = &tinyLen; + qparams[0].is_null = NULL; + qparams[0].num = 1; + + qparams[1].buffer_type = TSDB_DATA_TYPE_SMALLINT; + qparams[1].buffer_length = sizeof(v2); + qparams[1].buffer = &v2; + qparams[1].length = &smallLen; + qparams[1].is_null = NULL; + qparams[1].num = 1; + + taos_stmt_bind_param(stmt, qparams); + if (taos_stmt_execute(stmt) != 0) { + printf("\033[31mfailed to execute select statement.error:%s\033[0m\n", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + return; + } + + result = taos_stmt_use_result(stmt); + + TAOS_ROW row; + int rows = 0; + int num_fields = taos_num_fields(result); + TAOS_FIELD* fields = taos_fetch_fields(result); + + // fetch the records row by row + while ((row = taos_fetch_row(result))) { + char temp[256] = {0}; + rows++; + taos_print_row(temp, row, fields, num_fields); + printf("%s\n", temp); + } + + taos_free_result(result); + taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); +} + +void retrieve_callback(void* param, TAOS_RES* tres, int numOfRows) { + if (numOfRows > 0) { + printf("%d rows async retrieved\n", numOfRows); + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + if (numOfRows < 0) { + printf("\033[31masync retrieve failed, code: %d\033[0m\n", numOfRows); + } else { + printf("async retrieve completed\n"); + } + taos_free_result(tres); + } +} + +void select_callback(void* param, TAOS_RES* tres, int code) { + if (code == 0 && tres) { + taos_fetch_rows_a(tres, retrieve_callback, param); + } else { + printf("\033[31masync select failed, code: %d\033[0m\n", code); + } +} + +void verify_async(TAOS* taos) { + prepare_data(taos); + taos_query_a(taos, "select * from meters", select_callback, NULL); + usleep(1000000); +} + +void stream_callback(void* param, TAOS_RES* res, TAOS_ROW row) { + if (res == NULL || row == NULL) { + return; + } + + int num_fields = taos_num_fields(res); + TAOS_FIELD* fields = taos_fetch_fields(res); + + printf("got one row from stream_callback\n"); + char temp[256] = {0}; + taos_print_row(temp, row, fields, num_fields); + puts(temp); +} + +int main(int argc, char* argv[]) { + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* passwd = "taosdata"; + + taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); + TAOS* taos = taos_connect(host, user, passwd, "", 0); + if (taos == NULL) { + printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); + exit(1); + } + + const char* info = taos_get_server_info(taos); + printf("server info: %s\n", info); + info = taos_get_client_info(taos); + printf("client info: %s\n", info); + + printf("************ verify query *************\n"); + verify_query(taos); + + printf("********* verify async query **********\n"); + verify_async(taos); + + printf("************ verify prepare *************\n"); + verify_prepare(taos); + + printf("************ verify prepare2 *************\n"); + verify_prepare2(taos); + printf("************ verify prepare3 *************\n"); + verify_prepare3(taos); + + printf("done\n"); + taos_close(taos); + taos_cleanup(); +} diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index f48f438d9fa03e419f27d3c9b32e67db2b686d18..88dada44accf588357314843d2a7d09964c896c0 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -79,6 +79,9 @@ int64_t bpTs; #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) typedef struct { + bool singleTbInsert; + int32_t singleTbIdx; + int64_t* tsData; bool* boolData; int8_t* tinyData; @@ -116,6 +119,7 @@ int insertMBMETest4(TAOS_STMT *stmt, TAOS *taos); int insertMPMETest1(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest1(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); +int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); @@ -130,6 +134,7 @@ typedef struct { int32_t *colList; // full table column list int32_t testType; int32_t autoCreateTbl; + bool duplicateValue; bool fullCol; int32_t (*runFn)(TAOS_STMT*, TAOS*); int32_t tblNum; @@ -143,46 +148,47 @@ typedef struct { } CaseCfg; CaseCfg gCase[] = { - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, false, true, insertMBSETest1, 1, 10, 10, 0, 0, 0, 1, -1}, + {"insert:MBSE0-FULL", tListLen(shortColList), shortColList, TTYPE_INSERT, 0, false, true, insertMBSETest1, 10, 100, 10, 0, 0, 0, 1, -1}, - {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBSETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBSETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBSETest1, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBSE2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBSETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBSE2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBSETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBSE2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBSETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME1-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest1, 10, 10, 2, 2, 0, 0, 1, -1}, // 11 - {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME2-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBMETest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME2-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest2, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME2-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest2, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME3-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBMETest3, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME3-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest3, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME3-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest3, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, - {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, + {"insert:MBME4-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMBMETest4, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MBME4-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest4, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MBME4-C002", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMBMETest4, 10, 10, 2, 2, 0, 0, 1, -1}, - {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, + {"insert:MPME1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, true, insertMPMETest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:MPME1-C012", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, false, false, insertMPMETest1, 10, 10, 2, 12, 0, 0, 1, -1}, // 22 - {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 1, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, - {"insert:AUTO1-TBEXISTS", tListLen(fullColList), fullColList, TTYPE_INSERT, 3, true, insertAUTOTest2, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO1-FULL", tListLen(fullColList), fullColList, TTYPE_INSERT, 1, false, true, insertAUTOTest1, 10, 10, 2, 0, 0, 0, 1, -1}, + {"insert:AUTO2-TBEXISTS", tListLen(fullColList), fullColList, TTYPE_INSERT, 3, false, true, insertAUTOTest2, 10, 10, 2, 0, 0, 0, 1, -1}, +// {"insert:AUTO3-NTB", tListLen(fullColList), fullColList, TTYPE_INSERT, 0, true, true, insertAUTOTest3, 10, 10, 2, 0, 0, 0, 1, -1}, - {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, - {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, -// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, -// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, +// {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, }; @@ -219,7 +225,7 @@ typedef struct { int32_t caseRunNum; // total run case num } CaseCtrl; -#if 1 +#if 0 CaseCtrl gCaseCtrl = { .precision = TIME_PRECISION_MICRO, .bindNullNum = 0, @@ -244,7 +250,7 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = 0, + .caseIdx = 24, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, @@ -252,7 +258,7 @@ CaseCtrl gCaseCtrl = { #endif -#if 0 +#if 1 CaseCtrl gCaseCtrl = { // default .precision = TIME_PRECISION_MILLI, .bindNullNum = 0, @@ -382,7 +388,11 @@ bool colExists(TAOS_MULTI_BIND* pBind, int32_t dataType) { void generateInsertSQL(BindData *data) { int32_t len = 0; if (gCurCase->tblNum > 1) { - len = sprintf(data->sql, "insert into ? "); + if (data->singleTbInsert) { + len = sprintf(data->sql, "insert into %s%d ", bpTbPrefix, data->singleTbIdx); + } else { + len = sprintf(data->sql, "insert into ? "); + } } else { len = sprintf(data->sql, "insert into %s0 ", bpTbPrefix); } @@ -938,7 +948,14 @@ int32_t prepareInsertData(BindData *data) { } for (int32_t i = 0; i < allRowNum; ++i) { - data->tsData[i] = bpTs++; + if (gCurCase->duplicateValue) { + data->tsData[i] = bpTs; + if (i % 2 == 1) { + bpTs++; + } + } else { + data->tsData[i] = bpTs++; + } data->boolData[i] = (bool)(i % 2); data->tinyData[i] = (int8_t)i; data->utinyData[i] = (uint8_t)(i+1); @@ -1251,6 +1268,9 @@ void bpCheckParamNum(TAOS_STMT *stmt) { void bpCheckAffectedRows(TAOS_STMT *stmt, int32_t times) { int32_t rows = taos_stmt_affected_rows(stmt); int32_t insertNum = gCurCase->rowNum * gCurCase->tblNum * times; + if (gCurCase->duplicateValue) { + insertNum /= 2; + } if (insertNum != rows) { printf("!!!affected rows %d mis-match with insert num %d\n", rows, insertNum); exit(1); @@ -2014,6 +2034,65 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos) { return 0; } +/* normal table [prepare [bind add exec]] */ +int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos) { + int32_t loop = 0; + + while (gCurCase->bindColNum > 0) { + BindData data = {0}; + data.singleTbInsert = true; + prepareInsertData(&data); + + int32_t bindTimes = gCurCase->rowNum/gCurCase->bindRowNum; + for (int32_t t = 0; t< gCurCase->tblNum; ++t) { + data.singleTbIdx = t; + generateInsertSQL(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + for (int32_t b = 0; b bindColNum + b*gCurCase->bindColNum)) { + exit(1); + } + + if (taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + if (taos_stmt_execute(stmt) != 0) { + printf("!!!taos_stmt_execute error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + } + } + + bpCheckIsInsert(stmt, 1); + + destroyData(&data); + + gCurCase->bindColNum -= 2; + gCurCase->fullCol = false; + loop++; + } + + bpCheckAffectedRows(stmt, loop); + + gExecLoopTimes = loop; + + return 0; +} + /* select * from table */ int queryColumnTest(TAOS_STMT *stmt, TAOS *taos) { @@ -2160,7 +2239,7 @@ void prepareCheckResult(TAOS *taos, bool silent) { sprintf(buf, "%s%d", bpTbPrefix, 0); } - prepareCheckResultImpl(taos, buf, gCaseCtrl.printRes, gCurCase->rowNum * gExecLoopTimes, silent); + prepareCheckResultImpl(taos, buf, gCaseCtrl.printRes, gCurCase->duplicateValue ? (gCurCase->rowNum * gExecLoopTimes / 2) : (gCurCase->rowNum * gExecLoopTimes), silent); } gExecLoopTimes = 1; @@ -2805,6 +2884,7 @@ void runAll(TAOS *taos) { gCaseCtrl.bindColNum = 6; runCaseList(taos); gCaseCtrl.bindColNum = 0; + #endif /* diff --git a/examples/c/demoapi.c b/tests/script/api/demoapi.c similarity index 99% rename from examples/c/demoapi.c rename to tests/script/api/demoapi.c index 18f1b5a0592a27b1f40c5c7f7c64e6e7341d1a6e..6c060a6325dccc03f18e295dd88329df7e81395c 100644 --- a/examples/c/demoapi.c +++ b/tests/script/api/demoapi.c @@ -21,7 +21,6 @@ #ifndef WINDOWS #include #endif -#include "osSleep.h" #include "taos.h" #define debugPrint(fmt, ...) \ @@ -81,11 +80,9 @@ static void prepare_data(TAOS* taos) { TAOS_RES *res; res = taos_query(taos, "drop database if exists test;"); taos_free_result(res); - taosMsleep(100); res = taos_query(taos, "create database test;"); taos_free_result(res); - taosMsleep(100); if (taos_select_db(taos, "test")) { errorPrint("%s() LN%d: taos_select_db() failed\n", __func__, __LINE__); diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index 3983f533da139dac940f79b69813142501aa911f..b395047a1c724fd826015cb7cec8280a3b319fce 100755 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -122,6 +122,9 @@ function runSimCases() { function runPythonCases() { echo "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' cases.task cd $TDENGINE_DIR/tests/system-test runCasesOneByOne ../parallel_test/cases.task system-test diff --git a/tests/script/sh/compile_udf.sh b/tests/script/sh/compile_udf.sh index c7148d7d7d5200fe12cb2af32172dfd6b1d9c68a..5265e5a99b3400ac93702d7ee8a972f5d2893b33 100755 --- a/tests/script/sh/compile_udf.sh +++ b/tests/script/sh/compile_udf.sh @@ -1,10 +1,11 @@ set +e -rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so +rm -rf /tmp/udf/libbitand.so /tmp/udf/libsqrsum.so /tmp/udf/libgpd.so mkdir -p /tmp/udf echo "compile udf bit_and and sqr_sum" gcc -fPIC -shared sh/bit_and.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libbitand.so gcc -fPIC -shared sh/l2norm.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libl2norm.so +gcc -fPIC -shared sh/gpd.c -I../../include/libs/function/ -I../../include/client -I../../include/util -o /tmp/udf/libgpd.so echo "debug show /tmp/udf/*.so" ls /tmp/udf/*.so diff --git a/tests/script/sh/gpd.c b/tests/script/sh/gpd.c index 8d69bacb5edac29783ce860fb3a8dd5b407541d8..2259efa64a500cd7bf331642445c47a08883bec1 100644 --- a/tests/script/sh/gpd.c +++ b/tests/script/sh/gpd.c @@ -12,13 +12,10 @@ TAOS* taos = NULL; DLL_EXPORT int32_t gpd_init() { - taos = taos_connect("localhost", "root", "taosdata", "", 7100); return 0; } DLL_EXPORT int32_t gpd_destroy() { - taos_close(taos); - taos_cleanup(); return 0; } @@ -32,43 +29,18 @@ DLL_EXPORT int32_t gpd(SUdfDataBlock* block, SUdfColumn *resultCol) { SUdfColumnData *resultData = &resultCol->colData; resultData->numOfRows = block->numOfRows; for (int32_t i = 0; i < resultData->numOfRows; ++i) { - int j = 0; - for (; j < block->numOfCols; ++j) { - if (udfColDataIsNull(block->udfCols[j], i)) { - udfColDataSetNull(resultCol, i); - break; - } - } - if ( j == block->numOfCols) { - int32_t luckyNum = 88; - udfColDataSet(resultCol, i, (char *)&luckyNum, false); - } + int64_t* calc_ts = (int64_t*)udfColDataGetData(block->udfCols[0], i); + char* varTbname = udfColDataGetData(block->udfCols[1], i); + char* varDbname = udfColDataGetData(block->udfCols[2], i); + + char dbName[256] = {0}; + char tblName[256] = {0}; + memcpy(dbName, varDataVal(varDbname), varDataLen(varDbname)); + memcpy(tblName, varDataVal(varTbname), varDataLen(varTbname)); + printf("%s, %s\n", dbName, tblName); + int32_t result = 0; + udfColDataSet(resultCol, i, (char*)&result, false); } - TAOS_RES* res = taos_query(taos, "create database if not exists gpd"); - if (taos_errno(res) != 0) { - char* errstr = taos_errstr(res); - } - res = taos_query(taos, "create table gpd.st (ts timestamp, f int) tags(t int)"); - if (taos_errno(res) != 0) { - char* errstr = taos_errstr(res); - } - taos_query(taos, "insert into gpd.t using gpd.st tags(1) values(now, 1) "); - if (taos_errno(res) != 0) { - char* errstr = taos_errstr(res); - } - - taos_query(taos, "select * from gpd.t"); - if (taos_errno(res) != 0) { - char* errstr = taos_errstr(res); - } - - //to simulate actual processing delay by udf -#ifdef LINUX - usleep(1 * 1000); // usleep takes sleep time in us (1 millionth of a second) -#endif -#ifdef WINDOWS - Sleep(1); -#endif return 0; } diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 63018aea8c261532650cb1eed8a69667d5ccd509..f4392fbca4727a9a8b2adfba3b24ebf2a5c9ed1a 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -69,7 +69,7 @@ endi if $data4_db != 3 then # replica return -1 endi -if $data5_db != off then # strict +if $data5_db != on then # strict return -1 endi if $data6_db != 345600m then # duration diff --git a/tests/script/tsim/db/alter_replica_13.sim b/tests/script/tsim/db/alter_replica_13.sim index 4eafb8619857c1ca50d64f642e681ec4d956d15d..d75acb50ad087383fd2e5aabadfb0e2c1165204e 100644 --- a/tests/script/tsim/db/alter_replica_13.sim +++ b/tests/script/tsim/db/alter_replica_13.sim @@ -116,6 +116,25 @@ endi print ============= step4: alter database sql alter database db replica 3 +$wt = 0 +stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 $leaderIndex = 0 diff --git a/tests/script/tsim/db/alter_replica_31.sim b/tests/script/tsim/db/alter_replica_31.sim index 47e1fda79feaa221d0f8ee863214aa52160c6a38..74e800779151ac3768eca97818742723ce0f20f8 100644 --- a/tests/script/tsim/db/alter_replica_31.sim +++ b/tests/script/tsim/db/alter_replica_31.sim @@ -148,6 +148,26 @@ endi print ============= step3: alter database sql alter database db replica 1 +$wt = 0 + stepwt1: + $wt = $wt + 1 + sleep 1000 + if $wt == 200 then + print ====> dnode not ready! + return -1 + endi +sql show transactions +if $rows != 0 then + print wait 1 seconds to alter + goto stepwt1 +endi + +sql show db.vgroups +print ---> $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data27 $data28 $data29 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data26 $data37 $data38 $data39 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 $data36 $data47 $data48 $data49 + $hasleader = 0 $x = 0 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index 7012fbac6c0edf9000c030d297bfc79a99eed119..382cc986e241d3108268491ae1b0ca9ef1c51bdb 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -89,7 +89,7 @@ if $data4_db != 1 then # replica print expect 1, actual: $data4_db return -1 endi -if $data5_db != off then # strict +if $data5_db != on then # strict return -1 endi if $data6_db != 14400m then # duration diff --git a/tests/script/tsim/dnode/balance2.sim b/tests/script/tsim/dnode/balance2.sim index f677d3925cc50a7143c1296e01a37f19b2dfbef3..3f5a42d4d33b1c29acc054d816dc213c573108d0 100644 --- a/tests/script/tsim/dnode/balance2.sim +++ b/tests/script/tsim/dnode/balance2.sim @@ -4,11 +4,11 @@ system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 system sh/deploy.sh -n dnode5 -i 5 -system sh/cfg.sh -n dnode1 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode4 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode5 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode4 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode5 -c supportVnodes -v 5 print ========== step1 system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index 4c70399e4bd27e1a89848d74c8d8cde27d9e6a16..2e436945e00483a98886f3e72888cef0b58819ba 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -78,8 +78,8 @@ $ts2 = $tb2 . .ts print ===============================groupby_operation print -print ==== select count(*), c1 from group_tb0 group by c1 -sql select count(*), c1 from group_tb0 group by c1 +print ==== select count(*), c1 from group_tb0 group by c1 order by c1 +sql select count(*), c1 from group_tb0 group by c1 order by c1 print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -98,18 +98,18 @@ endi if $data90 != 10 then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data11 != 6 then +if $data11 != 1 then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi -print ==== select first(ts),c1 from group_tb0 group by c1; -sql select first(ts),c1 from group_tb0 group by c1; +print ==== select first(ts),c1 from group_tb0 group by c1 order by c1; +sql select first(ts),c1 from group_tb0 group by c1 order by c1; print rows: $rows print $data00 $data01 $data02 $data03 print $data10 $data11 $data12 $data13 @@ -120,16 +120,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.007@ then +if $data00 != @22-01-01 00:00:00.000@ then return -1 endi -if $data01 != 7 then +if $data01 != 0 then return -1 endi -if $data90 != @22-01-01 00:00:00.003@ then +if $data90 != @22-01-01 00:00:00.009@ then return -1 endi -if $data91 != 3 then +if $data91 != 9 then return -1 endi diff --git a/tests/script/tsim/parser/lastrow_query.sim b/tests/script/tsim/parser/lastrow_query.sim index a1b14c7a0e1a932a688581a38749a41d44690f0e..3af5ade10a2b2dd282f2a1e32cb8e1c22b68cd7b 100644 --- a/tests/script/tsim/parser/lastrow_query.sim +++ b/tests/script/tsim/parser/lastrow_query.sim @@ -70,10 +70,10 @@ sql select _wstart, t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00 if $row != 2 then return -1 endi -if $data01 != NULL then +if $data01 != 8 then return -1 endi -if $data02 != NULL then +if $data02 != 8 then return -1 endi if $data03 != NULL then diff --git a/tests/script/tsim/query/udf_with_const.sim b/tests/script/tsim/query/udf_with_const.sim new file mode 100644 index 0000000000000000000000000000000000000000..7a2a3389bd155e1c384823ce94c3fdd96b14139e --- /dev/null +++ b/tests/script/tsim/query/udf_with_const.sim @@ -0,0 +1,45 @@ +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 udf +system sh/compile_udf.sh +sql create database udf vgroups 3; +sql use udf; + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2)(now+2s,3)(now+3s,4)(now+4s,5)(now+5s,6)(now+6s,7); + +system_content printf %OS% +if $system_content == Windows_NT then + return 0; +endi + +if $system_content == Windows_NT then + sql create function gpd as 'C:\\Windows\\Temp\\gpd.dll' outputtype int bufSize 8; +else + sql create function gpd as '/tmp/udf/libgpd.so' outputtype int bufSize 8; +endi +sql show functions; +if $rows != 1 then + return -1 +endi + +sql select gpd(ts, tbname, 'detail') from t; +if $rows != 7 then + return -1 +endi +print $data00 $data10 +if $data00 != @0@ then + return -1 +endi +sql drop function gpd; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index bb2ea42383697936f1c07cde2b0e53ca38797f40..7bf10df6372fac03bdfc2cae6a9eef2d0c01b6a9 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -792,4 +792,46 @@ if $rows != 1 then goto loop17 endi +#for TS-2242 +sql create database test5 vgroups 1; +sql use test5; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create stream streams5 trigger at_once into streamt5 as select count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; +sql create stream streams6 trigger at_once into streamt6 as select count(*), _wstart, _wend, max(a), _wstart as ts from ts1 interval(10s) ; + +sql_error create stream streams7 trigger at_once into streamt7 as select _wstart, count(*), _wstart, _wend, max(a) from ts1 interval(10s) ; +sql_error create stream streams8 trigger at_once into streamt8 as select count(*), _wstart, _wstart, _wend, max(a) from ts1 interval(10s) ; +sql_error create stream streams9 trigger at_once into streamt9 as select _wstart as ts, count(*), _wstart as ts, _wend, max(a) from ts1 interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); + +print ====== test _wstart + +$loop_count = 0 + +loop17: + +sleep 200 +sql select * from streamt5; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop17 +endi + +sql select * from streamt6; + +if $rows != 1 then + print =====rows=$rows + goto loop17 +endi + +print ====== test _wstart end + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/drop_stream.sim b/tests/script/tsim/stream/drop_stream.sim index 817780ca59a009c083fbb18c31027cccc9109deb..1a474bd9aeca6f7f241f108ff14a0872a2eddf86 100644 --- a/tests/script/tsim/stream/drop_stream.sim +++ b/tests/script/tsim/stream/drop_stream.sim @@ -219,7 +219,7 @@ sql drop database test; print ========== interval\session\state window -sql CREATE DATABASE test1 BUFFER 96 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 STRICT 'off' WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0; +sql CREATE DATABASE test1 BUFFER 96 CACHESIZE 1 CACHEMODEL 'none' COMP 2 DURATION 14400m WAL_FSYNC_PERIOD 3000 MAXROWS 4096 MINROWS 100 KEEP 5256000m,5256000m,5256000m PAGES 256 PAGESIZE 4 PRECISION 'ms' REPLICA 1 WAL_LEVEL 1 VGROUPS 2 SINGLE_STABLE 0; sql use test1; sql CREATE STABLE st (time TIMESTAMP, ca DOUBLE, cb DOUBLE, cc int) TAGS (ta VARCHAR(10) ); diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index ee6eefde2633f6422dc51927421ef109392dc2ae..f535fd619fbedfa87598569ba7edfde82bd18086 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -197,4 +197,98 @@ if $data01 != 1 then return -1 endi +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams3 trigger at_once into streamt3 as select _wstart, count(*) c1 from t1 where a > 5 session(ts, 5s); +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 +loop13: +sleep 200 + +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 0 then + print =====rows=$rows + goto loop13 +endi + +sql insert into t1 values(1648791213000,6,2,3,1.0); + +$loop_count = 0 +loop14: +sleep 200 +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 +loop15: +sleep 200 +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop15 +endi + + +sql insert into t1 values(1648791223000,2,2,3,1.0); +sql insert into t1 values(1648791223000,10,2,3,1.0); +sql insert into t1 values(1648791233000,10,2,3,1.0); + +$loop_count = 0 +loop16: +sleep 200 +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop16 +endi + +sql insert into t1 values(1648791233000,2,2,3,1.0); + +$loop_count = 0 +loop17: +sleep 200 +sql select * from streamt3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop17 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index 49e22db1b12292f5c6a2b1c8141e9330c7fc6c20..c9a1ddd922b4c6d1194b87902ef6bdf22826165e 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -461,7 +461,7 @@ sql insert into t2 values(1648791213004,4,10,10,4.1); $loop_count = 0 loop2: -sleep 100 +sleep 200 $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -519,7 +519,7 @@ print step 6 $loop_count = 0 loop3: -# sleep 300 +sleep 300 $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -618,6 +618,60 @@ if $data41 != 2 then goto loop4 endi +sql insert into t1 values(1648791343003,4,4,4,3.1); +sql insert into t1 values(1648791213004,4,5,5,4.1); + +loop5: +sleep 200 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt3; + +# row 0 +if $rows != 7 then + print =====rows=$rows + goto loop5 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop5 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop5 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop5 +endi + +if $data31 != 4 then + print =====data31=$data31 + goto loop5 +endi + +if $data41 != 2 then + print =====data41=$data41 + goto loop5 +endi + +if $data51 != 1 then + print =====data51=$data51 + goto loop5 +endi + +if $data61 != 1 then + print =====data61=$data61 + goto loop5 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all diff --git a/tests/script/tsim/stream/state0.sim b/tests/script/tsim/stream/state0.sim index 7c922658c91cd3c6529b981208b31754226db71d..ea7528b1cb3d2ca33e805bb0e516168a26d8ef59 100644 --- a/tests/script/tsim/stream/state0.sim +++ b/tests/script/tsim/stream/state0.sim @@ -552,7 +552,7 @@ sql use test4; sql create table st (ts timestamp, c1 tinyint, c2 smallint) tags (t1 tinyint) ; sql create table t1 using st tags (-81) ; sql create table t2 using st tags (-81) ; -sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS start, min(c1),count(c1) from t1 state_window(c1); +sql create stream if not exists streams4 trigger window_close into streamt4 as select _wstart AS startts, min(c1),count(c1) from t1 state_window(c1); sql insert into t1 (ts, c1) values (1668073288209, 11); sql insert into t1 (ts, c1) values (1668073288210, 11); @@ -567,7 +567,7 @@ loop7: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -606,7 +606,7 @@ loop8: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -640,7 +640,7 @@ loop8: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then @@ -679,7 +679,7 @@ loop9: sleep 200 -sql select * from streamt4 order by start; +sql select * from streamt4 order by startts; $loop_count = $loop_count + 1 if $loop_count == 20 then diff --git a/tests/script/tsim/user/privilege_db.sim b/tests/script/tsim/user/privilege_db.sim index 79e3754c2fe64b0978c172c6acec17becefe5cb8..b708fdab64c1931d6a86c67a8f3ec86cb880677a 100644 --- a/tests/script/tsim/user/privilege_db.sim +++ b/tests/script/tsim/user/privilege_db.sim @@ -6,8 +6,8 @@ sql connect print =============== create db sql create database d1 vgroups 1; sql use d1 -sql create table stb (ts timestamp, i int) tags (j int) -sql create topic topic_1 as select ts, i from stb +sql create table d1_stb (ts timestamp, i int) tags (j int) +sql create topic d1_topic_1 as select ts, i from d1_stb sql create database d2 vgroups 1; sql create database d3 vgroups 1; @@ -72,7 +72,6 @@ sql REVOKE read,write ON d1.* from user1; sql REVOKE read,write ON d2.* from user1; sql REVOKE read,write ON *.* from user1; - print =============== create users sql create user u1 PASS 'taosdata' sql select * from information_schema.ins_users @@ -92,7 +91,17 @@ sql_error drop database d1; sql_error drop database d2; sql_error create stable d1.st (ts timestamp, i int) tags (j int) -sql create stable d2.st (ts timestamp, i int) tags (j int) -sql_error create topic topic_2 as select ts, i from stb +sql use d2 +sql create table d2_stb (ts timestamp, i int) tags (j int) + +# Insufficient privilege for operation +sql_error create topic d2_topic_1 as select ts, i from d2_stb + +sql use d1 +# Insufficient privilege for operation +sql_error drop topic d1_topic_1 +sql create topic d1_topic_2 as select ts, i from d1_stb + +sql drop topic d1_topic_2 system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/user/privilege_topic.sim b/tests/script/tsim/user/privilege_topic.sim new file mode 100644 index 0000000000000000000000000000000000000000..9ce5bebec3d68761fb5364896f81e4a5e1428878 --- /dev/null +++ b/tests/script/tsim/user/privilege_topic.sim @@ -0,0 +1,156 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create db +sql create database root_d1 vgroups 1; +sql create database root_d2 vgroups 1; +sql create database root_d3 vgroups 1; + +sql show user privileges +if $rows != 1 then + return -1 +endi +if $data(root)[1] != all then + return -1 +endi +if $data(root)[2] != all then + return -1 +endi + +print =============== create users +sql create user user1 PASS 'taosdata' +sql create user user2 PASS 'taosdata' +sql create user user3 PASS 'taosdata' +sql create user user4 PASS 'taosdata' +sql create user user5 PASS 'taosdata' +sql create user user6 PASS 'taosdata' +sql alter user user1 sysinfo 0 + +sql select * from information_schema.ins_users +if $rows != 7 then + return -1 +endi + +sql GRANT read ON root_d1.* to user1; +sql GRANT write ON root_d2.* to user2; +sql GRANT read ON root_d3.* to user3; +sql GRANT write ON root_d3.* to user3; + +sql show user privileges +if $rows != 5 then + return -1 +endi +if $data(user1)[1] != read then + return -1 +endi +if $data(user1)[2] != root_d1 then + return -1 +endi +if $data(user2)[1] != write then + return -1 +endi +if $data(user2)[2] != root_d2 then + return -1 +endi + +print =============== create topis +sql use root_d1 +sql create table root_d1_stb (ts timestamp, i int) tags (j int) +sql create topic root_d1_topic1 as select ts, i from root_d1_stb +sql create topic root_d1_topic2 as select ts, i from root_d1_stb +sql create topic root_d1_topic3 as select ts, i from root_d1_stb +sql create topic root_d1_topic4 as select ts, i from root_d1_stb + +sql show user privileges +if $rows != 5 then + return -1 +endi + +sql GRANT subscribe ON root_d1_topic1 TO user4 +sql GRANT subscribe ON root_d1_topic2 TO user5 +sql GRANT subscribe ON root_d1_topic3 TO user6 +sql show user privileges +if $rows != 8 then + return -1 +endi + +if $data(user4)[1] != subscribe then + return -1 +endi +if $data(user4)[2] != root_d1_topic1 then + return -1 +endi +if $data(user5)[1] != subscribe then + return -1 +endi +if $data(user5)[2] != root_d1_topic2 then + return -1 +endi +if $data(user6)[1] != subscribe then + return -1 +endi +if $data(user6)[2] != root_d1_topic3 then + return -1 +endi + +sql REVOKE subscribe ON root_d1_topic3 from user6 +sql show user privileges +if $rows != 7 then + return -1 +endi +if $data(user4)[1] != subscribe then + return -1 +endi +if $data(user4)[2] != root_d1_topic1 then + return -1 +endi +if $data(user5)[1] != subscribe then + return -1 +endi +if $data(user5)[2] != root_d1_topic2 then + return -1 +endi + +print =============== repeat revoke/grant or invalid revoke/grant +sql GRANT subscribe ON root_d1_topic1 to user4 +sql GRANT subscribe ON root_d1_topic2 to user4 +sql GRANT subscribe ON root_d1_topic3 to user4 +sql GRANT subscribe ON root_d1_topic1 to user5 +sql GRANT subscribe ON root_d1_topic2 to user5 +sql GRANT subscribe ON root_d1_topic3 to user5 +sql GRANT subscribe ON root_d1_topic1 to user6 +sql GRANT subscribe ON root_d1_topic2 to user6 +sql GRANT subscribe ON root_d1_topic3 to user6 +sql REVOKE subscribe ON root_d1_topic1 from user4 +sql REVOKE subscribe ON root_d1_topic2 from user4 +sql REVOKE subscribe ON root_d1_topic3 from user4 +sql REVOKE subscribe ON root_d1_topic1 from user5 +sql REVOKE subscribe ON root_d1_topic2 from user5 +sql REVOKE subscribe ON root_d1_topic3 from user5 +sql REVOKE subscribe ON root_d1_topic1 from user6 +sql REVOKE subscribe ON root_d1_topic2 from user6 +sql REVOKE subscribe ON root_d1_topic3 from user6 + +print =============== invalid revoke/grant +sql_error GRANT subscribe ON root_d1_topicx from user5 +sql_error REVOKE subscribe ON root_d1_topicx from user5 + +print =============== check +sql GRANT subscribe ON root_d1_topic1 TO user4 +sql GRANT subscribe ON root_d1_topic2 TO user5 +sql GRANT subscribe ON root_d1_topic3 TO user6 +sql show user privileges +if $rows != 8 then + return -1 +endi + +print =============== re connect +print user u1 login +sql close +sql connect user1 + +sql_error show user privileges + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index 594d649714bf6524d807d996c6f8fa86fcf967d6..caf316bd868a057cd8a161817c85d8664a9db82e 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -121,12 +121,13 @@ if $data01 != 152.420471066 then return -1 endi -sql select udf2(f2) from udf.t2 group by 1-udf1(f1); +sql select udf2(f2) from udf.t2 group by 1-udf1(f1) order by 1-udf1(f1) print $rows , $data00 , $data10 if $rows != 2 then return -1 endi if $data00 != 2.000000000 then + print expect 2.000000000 , actual: $data00 return -1 endi if $data10 != 12.083045974 then diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index 8efba515ae5778afdf6cd3f36079d4426741601d..10e18d01d07eda4957fa98050496d4b06f81126a 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -85,6 +85,7 @@ print ======== step3 system sh/exec.sh -n dnode2 -s stop sleep 3000 +$t = 0 $x = 0 loop: @@ -126,8 +127,8 @@ print ======== step8 $lastRows = $data00 print ======== loop Times $x -if $x < 2 then - $x = $x + 1 +if $t < 2 then + $t = $t + 1 goto loop endi @@ -138,4 +139,4 @@ system sh/exec.sh -n dnode4 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s stop -x SIGINT system sh/exec.sh -n dnode6 -s stop -x SIGINT system sh/exec.sh -n dnode7 -s stop -x SIGINT -system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode8 -s stop -x SIGINT diff --git a/tests/script/tsim/vnode/stable_balance_replica1.sim b/tests/script/tsim/vnode/stable_balance_replica1.sim index f1249793975e8fc82692549c259a6289f3091ede..84190a3be09ae075ec136157e2b2c97765a28c2d 100644 --- a/tests/script/tsim/vnode/stable_balance_replica1.sim +++ b/tests/script/tsim/vnode/stable_balance_replica1.sim @@ -3,10 +3,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode2 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode3 -c supportVnodes -v 4 -system sh/cfg.sh -n dnode4 -c supportVnodes -v 4 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode2 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode3 -c supportVnodes -v 5 +system sh/cfg.sh -n dnode4 -c supportVnodes -v 5 $dbPrefix = br1_db $tbPrefix = br1_tb diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index e3095e8b934cabeda50ad9b7976a5418512cef6a..5c7bb0443a282c4be28cfaa359eb8f41fb4c4ce4 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -62,8 +62,8 @@ def taos_command (buildPath, key, value, expectString, cfgDir, sqlString='', key print ('taos login success! Here can run sql, taos> ') if len(sqlString) != 0: child.sendline (sqlString) - w = child.expect(["Query OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) - if w == 0: + w = child.expect(["Query OK", "Create OK", "Insert OK", "Drop OK", taosExpect.TIMEOUT, taosExpect.EOF], timeout=10) + if w == 0 or w == 1 or w == 2: return "TAOS_OK" else: print(1) @@ -233,7 +233,7 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -s") newDbName="dbss" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s fail") @@ -246,17 +246,17 @@ class TDTestCase: tdLog.exit("create db fail after taos -s %s fail"%(keyDict['s'])) keyDict['s'] = "\"create table " + newDbName + ".stb (ts timestamp, c int) tags (t int)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"create table " + newDbName + ".ctb0 using " + newDbName + ".stb tags (0) " + newDbName + ".ctb1 using " + newDbName + ".stb tags (1)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ctb0 values('2021-04-01 08:00:00.000', 10)('2021-04-01 08:00:01.000', 20) " + newDbName + ".ctb1 values('2021-04-01 08:00:00.000', 11)('2021-04-01 08:00:01.000', 21)\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -s insert data fail") @@ -401,27 +401,27 @@ class TDTestCase: tdLog.printNoPrefix("================================ parameter: -w") newDbName="dbw" keyDict['s'] = "\"create database " + newDbName + "\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w fail") keyDict['s'] = "\"create table " + newDbName + ".ntb (ts timestamp, c binary(128))\"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Create OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w create table fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.001', 'abcd0123456789')('2021-04-01 08:00:00.002', 'abcd012345678901234567890123456789') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.003', 'aaaaaaaaaaaaaaaaaaaa')('2021-04-01 08:00:01.004', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") keyDict['s'] = "\"insert into " + newDbName + ".ntb values('2021-04-01 08:00:00.005', 'cccccccccccccccccccc')('2021-04-01 08:00:01.006', 'dddddddddddddddddddddddddddddddddddddddd') \"" - retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", keyDict['c'], '', '', '') if retCode != "TAOS_OK": tdLog.exit("taos -w insert data fail") diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 7ad7e4d0ef90b7f30f692cb40596c14b653e903d..2125b5eebe448b4ac044158f7157cd689eaec29e 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -136,7 +136,7 @@ class TDTestCase: tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") - tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);") + tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS startts, min(k), max(k), sum(k) from source_db.stb interval(10m);") #TD-19944 -Q=3 diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index 862edbdde9b795f151b9d17f3c74abc8ebc4de63..488dfebff50c77d1dd08c6d4e46fdbf3bbe3ea7f 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -108,7 +108,7 @@ class TDTestCase: # create stream - tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') + tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') # insert data for i in range(num_random*n): @@ -187,20 +187,20 @@ class TDTestCase: sleep(5) # stream data check - tdSql.query("select start,wend,max_int from stream_max_stable_1 ;") + tdSql.query("select startts,wend,max_int from stream_max_stable_1 ;") tdSql.checkRows(20) tdSql.query("select sum(max_int) from stream_max_stable_1 ;") stream_data_1 = tdSql.queryResult[0][0] tdSql.query("select sum(min_int) from stream_max_stable_1 ;") stream_data_2 = tdSql.queryResult[0][0] - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] self.stream_value_check(stream_data_1,sql_data_1) self.stream_value_check(stream_data_2,sql_data_2) - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] diff --git a/tests/system-test/1-insert/drop.py b/tests/system-test/1-insert/drop.py index f8796bcf6aa0bd11bd1ddc840e51c973765b8df9..c15a9bbc35bc28a8679b3400ff37569e6d7955aa 100644 --- a/tests/system-test/1-insert/drop.py +++ b/tests/system-test/1-insert/drop.py @@ -132,7 +132,7 @@ class TDTestCase: tdSql.execute(f'drop database {self.dbname}') def drop_stream_check(self): - tdSql.execute(f'create database {self.dbname} replica {self.replicaVar}') + tdSql.execute(f'create database {self.dbname} replica 1') tdSql.execute(f'use {self.dbname}') stbname = tdCom.getLongName(5,"letters") stream_name = tdCom.getLongName(5,"letters") @@ -158,4 +158,4 @@ class TDTestCase: tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 3e2bec130ef50181c58537abdd5abafe9f3d9c32..8780ecc477f6c6a9ec1076967aa27da63eb09f97 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -147,19 +147,20 @@ class TDTestCase: def __current_cfg(self): cfg_list = [] current_case1 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}1 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}2 {L2} {NON_PRIMARY_DIR}" ] - current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + #current_case2 = [f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 {L0} {PRIMARY_DIR}"] + current_case2 = [] for i in range(9): current_case2.append(f"dataDir {self.taos_data_dir}/{DATA_PRE0}{i+1} {L0} {NON_PRIMARY_DIR}") # TD-17773bug current_case3 = [ - f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", + #f"dataDir {self.taos_data_dir}/{DATA_PRE0}0 ", f"dataDir {self.taos_data_dir}/{DATA_PRE0}1 {L0} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE1}0 {L1} {NON_PRIMARY_DIR}", f"dataDir {self.taos_data_dir}/{DATA_PRE2}0 {L2} {NON_PRIMARY_DIR}", diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 44243fe029233dc3458a7b6822600bc198e11824..93f3c7410df5bfc0ffef836d8353249f8de0c965 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -1719,6 +1719,7 @@ class TDTestCase: print(err.errno) def runAll(self): + """ for value_type in ["obj", "default"]: self.initCheckCase(value_type) self.symbolsCheckCase(value_type) @@ -1771,7 +1772,7 @@ class TDTestCase: # self.sStbStbDdataDtsMtInsertMultiThreadCheckCase() # self.sStbDtbDdataDtsMtInsertMultiThreadCheckCase() # self.lengthIcreaseCrashCheckCase() - + """ def run(self): print("running {}".format(__file__)) self.createDb() diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index f58882720610b32794fc48e4cd652ae70e63b5d7..a17a2d9514a50a06f594fca2990048b1876ef441 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -1416,8 +1416,8 @@ class TDTestCase: self.symbolsCheckCase() self.tsCheckCase() self.openTstbTelnetTsCheckCase() - self.idSeqCheckCase() - self.idLetterCheckCase() + #self.idSeqCheckCase() + #self.idLetterCheckCase() self.noIdCheckCase() self.maxColTagCheckCase() self.stbTbNameCheckCase() @@ -1450,7 +1450,7 @@ class TDTestCase: self.spellCheckCase() self.pointTransCheckCase() self.defaultTypeCheckCase() - self.tbnameTagsColsNameCheckCase() + #self.tbnameTagsColsNameCheckCase() # # # MultiThreads # self.stbInsertMultiThreadCheckCase() # self.sStbStbDdataInsertMultiThreadCheckCase() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index b764edebd732ad7d35fea98f0e75c08307991ff9..8ad1982b55506621606a92840e5496f29094da75 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -18,7 +18,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file def checkFileContent(self, dbname="sml_db"): @@ -76,13 +76,14 @@ class TDTestCase: tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) + tdSql.checkData(0, 2, "web02") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkData(1, 2, "web01") + tdSql.checkData(1, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 32d6ef98e99a1d4ebf9650d20c5aa3968fcf9dd8..a59180c2b19a039876c5b2a5407c866234075d03 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -30,7 +30,7 @@ class TDTestCase: self.stbname = f'{self.dbname}.stb' self.ctbname = f'{self.dbname}.ctb' - def check_ms_timestamp(self,unit,date_time): + def check_ms_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1a': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) @@ -50,13 +50,17 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) - tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000) + if ignore_tz == 0: + tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000) % (86400 * 1000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000) - def check_us_timestamp(self,unit,date_time): + def check_us_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1u': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) @@ -80,13 +84,17 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) - tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 ) + if ignore_tz == 0: + tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 ) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000) - def check_ns_timestamp(self,unit,date_time): + def check_ns_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1b': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': @@ -114,21 +122,26 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': - tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 ) + if ignore_tz == 0: + tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 ) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000) - def check_tb_type(self,unit,tb_type): + def check_tb_type(self,unit,tb_type,ignore_tz): if tb_type.lower() == 'ntb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ntbname}') elif tb_type.lower() == 'ctb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ctbname}') elif tb_type.lower() == 'stb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.stbname}') def data_check(self,date_time,precision,tb_type): + tz_options = [0, 1] for unit in self.time_unit: if (unit.lower() == '1u' and precision.lower() == 'ms') or (unit.lower() == '1b' and precision.lower() == 'us') or (unit.lower() == '1b' and precision.lower() == 'ms'): if tb_type.lower() == 'ntb': @@ -138,17 +151,20 @@ class TDTestCase: elif tb_type.lower() == 'stb': tdSql.error(f'select timetruncate(ts,{unit}) from {self.stbname}') elif precision.lower() == 'ms': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_ms_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type,ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_ms_timestamp(unit,date_time,ignore_tz) elif precision.lower() == 'us': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_us_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type,ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_us_timestamp(unit,date_time,ignore_tz) elif precision.lower() == 'ns': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_ns_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type, ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_ns_timestamp(unit,date_time,ignore_tz) for unit in self.error_unit: if tb_type.lower() == 'ntb': tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}') @@ -181,8 +197,8 @@ class TDTestCase: date_time = self.get_time.time_transform(self.ts_str,precision) self.data_check(date_time,precision,'ctb') self.data_check(date_time,precision,'stb') - - + + def run(self): self.function_check_ntb() self.function_check_stb() diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index 2b0336d2d7bff3d601abcde3f0c0e59f390e6b6b..6af9b130ef7b36c493e7bb1edc41fa2f391bf2e0 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -429,10 +429,10 @@ class TDTestCase: tdSql.checkRows(2) # nest query - tdSql.query(f"select unique(c1) from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) ") + tdSql.query(f"select unique(c1) v from (select _rowts , t1 ,c1 , tbname from {dbname}.stb1 ) order by v") tdSql.checkRows(11) - tdSql.checkData(0,0,6) - tdSql.checkData(10,0,3) + tdSql.checkData(1,0,0) + tdSql.checkData(10,0,9) tdSql.query(f"select unique(t1) from (select _rowts , t1 , tbname from {dbname}.stb1 )") tdSql.checkRows(2) tdSql.checkData(0,0,4) diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index b576b37a4d2dfb7ed099f4fca586c57a742153e9..d0157ec7c772182a2e24064dfec3cc29ed2a7ed6 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -125,7 +125,7 @@ class TDTestCase: tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') tdSql.query('select * from information_schema.ins_databases;') - tdSql.checkData(2,5,'off') + tdSql.checkData(2,5,'on') tdSql.error("alter database db strict 'off'") # tdSql.execute('alter database db strict 'on'') # tdSql.query('select * from information_schema.ins_databases;') diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 118a6caf7a6f6b3dae0a1467d7481809b68dbfa3..986806fdd83a7b2ee9d2b72dbfcf332c713323df 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -113,6 +113,13 @@ int32_t shellRunSingleCommand(char *command) { } void shellRecordCommandToHistory(char *command) { + if (strncasecmp(command, "create user ", 12) == 0 || strncasecmp(command, "alter user ", 11) == 0) { + if (taosStrCaseStr(command, " pass ")) { + // have password command forbid record to history because security + return; + } + } + SShellHistory *pHistory = &shell.history; if (pHistory->hstart == pHistory->hend || pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || @@ -135,7 +142,7 @@ int32_t shellRunCommand(char *command, bool recordHistory) { } // add help or help; - if(strcmp(command, "help") == 0 || strcmp(command, "help;") == 0) { + if(strncasecmp(command, "help;", 5) == 0) { showHelp(); return 0; } @@ -214,6 +221,18 @@ void shellRunSingleCommandImp(char *command) { return; } + // pre string + char * pre = "Query OK"; + if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Delete OK"; + } else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Insert OK"; + } else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Create OK"; + } else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { + pre = "Drop OK"; + } + TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands int32_t error_no = 0; @@ -229,10 +248,10 @@ void shellRunSingleCommandImp(char *command) { } taos_free_result(pSql); } else { - int32_t num_rows_affacted = taos_affected_rows(pSql); + int64_t num_rows_affacted = taos_affected_rows64(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); - printf("Query OK, %d row(s) affected (%.6fs)\r\n", num_rows_affacted, (et - st) / 1E6); + printf("%s, %" PRId64 " row(s) affected (%.6fs)\r\n", pre, num_rows_affacted, (et - st) / 1E6); // call auto tab callbackAutoTab(command, NULL, false); diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 94bb909e296cb791538301abae42e01fb704b448..bbb127b12846f113ee1583940ee8d005623bf22b 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -37,7 +37,9 @@ static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { const void* data = NULL; int rows; ws_fetch_block(wres, &data, &rows); - *execute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *execute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -77,7 +79,9 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -129,7 +133,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { taosCloseFile(&pFile); return 0; @@ -223,17 +229,23 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is down, will try to reconnect\n"); shell.ws_conn = NULL; } ws_free_result(res); - if (reconnectNum == 0) continue; + if (reconnectNum == 0) { + continue; + } else { + fprintf(stderr, "TDengine server is disconnected, will try to reconnect\n"); + } return; } break; } - double execute_time = ws_take_timing(res)/1E6; + double execute_time = 0; + if (res) { + execute_time = ws_take_timing(res)/1E6; + } if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\r\n\r\n"); diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 47b7adbf189a97b4be881b4d48fc499c4b9b7dbd..49885c0aea8098b9ea8160b0cd89c93d1f5b20aa 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -20,6 +20,7 @@ #include #include "taos.h" #include "types.h" +#include "tlog.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -77,11 +78,20 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + char *sql[4] = {0}; + sql[0] = taosMemoryCalloc(1, 128); + sql[1] = taosMemoryCalloc(1, 128); + sql[2] = taosMemoryCalloc(1, 128); + sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", "sys.if.bytes.out 1479496102 1.3E3 network=tcp", " sys.procs.running 1479496100 42 host=web01 "}; + for(int i = 0; i < 4; i++){ + strncpy(sql[i], sql1[i], 128); + } + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -146,28 +156,7 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - -int smlProcess_json4_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}}]" }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); @@ -676,52 +665,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -761,214 +704,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1003,10 +738,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { @@ -1017,7 +752,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1091,9 +826,10 @@ int sml_ts2164_Test() { taos_free_result(pRes); const char *sql[] = { +// "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", +// "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); @@ -1118,6 +854,9 @@ int sml_ttl_Test() { const char *sql[] = { "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833339000000", + }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -1127,6 +866,11 @@ int sml_ttl_Test() { printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); @@ -1140,7 +884,46 @@ int sml_ttl_Test() { return code; } +//char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +//}; +//uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +//} + int main(int argc, char *argv[]) { + +// for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ +// printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); +// } int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); @@ -1153,11 +936,9 @@ int main(int argc, char *argv[]) { ret = smlProcess_json1_Test(); ASSERT(!ret); ret = smlProcess_json2_Test(); - ASSERT(!ret); + ASSERT(ret); ret = smlProcess_json3_Test(); - ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); + ASSERT(ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1166,12 +947,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test();