diff --git a/.gitignore b/.gitignore index a69d2e44bd7eadf27d721ccf13c47ea91f6164ba..a6e222d2e9fba615319e7b03825e16c381313a13 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,4 @@ tools/COPYING tools/BUGS tools/taos-tools tools/taosws-rs +tags diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 5a0e7972c6081a3ff1c5e01a932c24709bc153a9..9c28489a39dbb843dfd95e583b9d3b9a1b9046ae 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -173,7 +173,7 @@ def pre_test_build_mac() { ''' sh ''' cd ${WK}/debug - cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false + cmake .. -DBUILD_TEST=true -DBUILD_HTTPS=false -DCMAKE_BUILD_TYPE=Release make -j10 ctest -j10 || exit 7 ''' diff --git a/README-CN.md b/README-CN.md index b5a2564f9101593335933b8ab2bdffd88e67014d..189b7a059aaebef7c05321220b668f5a64c79651 100644 --- a/README-CN.md +++ b/README-CN.md @@ -104,6 +104,16 @@ sudo yum install -y zlib-devel zlib-static xz-devel snappy-devel jansson jansson sudo yum config-manager --set-enabled Powertools ``` +#### CentOS + devtoolset + +除上述编译依赖包,需要执行以下命令: + +``` +sudo yum install centos-release-scl +sudo yum install devtoolset-9 devtoolset-9-libatomic-devel +scl enable devtoolset-9 -- bash +``` + ### macOS ``` diff --git a/README.md b/README.md index 3f7208dfb96d857c22f3b7fa83df6d25b4c46246..3b7e978a17acf51f4d204720eb45e14c56c77cfa 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,16 @@ If the PowerTools installation fails, you can try to use: sudo yum config-manager --set-enabled powertools ``` +#### For CentOS + devtoolset + +Besides above dependencies, please run following commands: + +``` +sudo yum install centos-release-scl +sudo yum install devtoolset-9 devtoolset-9-libatomic-devel +scl enable devtoolset-9 -- bash +``` + ### macOS ``` diff --git a/cmake/cmake.define b/cmake/cmake.define index 1b9a06d9e4abe516c68b47e7d7dff20538f2fc13..10f217254152db82518099560ef902b2ebabf499 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,7 @@ cmake_minimum_required(VERSION 3.0) set(CMAKE_VERBOSE_MAKEFILE OFF) +set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) diff --git a/cmake/cmake.install b/cmake/cmake.install index fd1e080ddab1478f73689e7cced405ae8404fbc2..67634625ce53c6ea48dc03746a39fdb03975c2af 100644 --- a/cmake/cmake.install +++ b/cmake/cmake.install @@ -21,7 +21,7 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.bat") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") - INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER})") + INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} :needAdmin ${TD_SOURCE_DIR} ${PROJECT_BINARY_DIR} Windows ${TD_VER_NUMBER} ${TD_BUILD_TAOSA_INTERNAL})") ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_SOURCE_DIR}/packaging/tools/make_install.sh") INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") diff --git a/cmake/cmake.platform b/cmake/cmake.platform index a4bfcaf609cc98f43094dabd993b8e310b9b220f..61eb66f675bd12840d025b5d0ed43192120506b4 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -37,6 +37,21 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin SET(TD_LINUX_32 TRUE) ENDIF () + EXECUTE_PROCESS(COMMAND chmod 777 ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh) + EXECUTE_PROCESS(COMMAND readlink /bin/sh OUTPUT_VARIABLE SHELL_LINK) + MESSAGE(STATUS "The shell is: " ${SHELL_LINK}) + + IF (${SHELL_LINK} MATCHES "dash") + EXECUTE_PROCESS(COMMAND ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) + ELSE () + EXECUTE_PROCESS(COMMAND sh ${CMAKE_CURRENT_LIST_DIR}/../packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) + ENDIF () + MESSAGE(STATUS "The current OS is " ${TD_OS_INFO}) + IF (${TD_OS_INFO} MATCHES "Alpine") + SET(TD_ALPINE TRUE) + ADD_DEFINITIONS("-D_ALPINE") + ENDIF () + ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") SET(TD_DARWIN TRUE) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index e79c6b799f0bfdf98e73d33188c12d355eb32708..b013d45911988ce38cd752766e862da3d461b1cd 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG f406d51 + GIT_TAG main SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/01-index.md b/docs/en/01-index.md index 13552ea9dcfbae71a368cad9d55b4a0030ac266a..296cd968986d5dd7738d6c3b997901614f893763 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -1,6 +1,7 @@ --- title: TDengine Documentation sidebar_label: Documentation Home +description: This website contains the user manuals for TDengine, an open-source, cloud-native time-series database optimized for IoT, Connected Cars, and Industrial IoT. slug: / --- diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index a60bfab2ccd30fb585d72ee14054029b1c176a66..95dd4324f145800c139272381d2a07a40a2b5ce9 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -1,5 +1,6 @@ --- title: Introduction +description: This document introduces the major features, competitive advantages, typical use cases, and benchmarks of TDengine. toc_max_heading_level: 2 --- diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md index 0b1b226c17100d56313b5480e26f437841afe8c7..771a061c310bd53c485394fe3a8897567b12e48b 100644 --- a/docs/en/04-concept/index.md +++ b/docs/en/04-concept/index.md @@ -1,5 +1,6 @@ --- title: Concepts +description: This document describes the basic concepts of TDengine, including the supertable. --- In order to explain the basic concepts and provide some sample code, the TDengine documentation smart meters as a typical time series use case. We assume the following: 1. Each smart meter collects three metrics i.e. current, voltage, and phase; 2. There are multiple smart meters; 3. Each meter has static attributes like location and group ID. Based on this, collected data will look similar to the following table: diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index ac273daba48d39bd0777ffdb30f431c6521d13b3..42e6861674434ad9800f69dcf00fe80a140e9bf7 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -1,6 +1,7 @@ --- -sidebar_label: Docker title: Quick Install on Docker +sidebar_label: Docker +description: This document describes how to install TDengine in a Docker container and perform queries and inserts. --- This document describes how to install TDengine in a Docker container and perform queries and inserts. diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 54d2e046c214f2df8ecd5ff03b7d5108c701938d..a0c1d939831395da596ecf15247c46b5a5188a14 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -1,6 +1,7 @@ --- -sidebar_label: Package title: Quick Install from Package +sidebar_label: Package +description: This document describes how to install TDengine on Linux, Windows, and macOS and perform queries and inserts. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/05-get-started/index.md b/docs/en/05-get-started/index.md index 12cfa22c69b1db8e82de4cb251ca3d8c67fc4546..66573a89cd6e181192132cc0b304f415fa25b89c 100644 --- a/docs/en/05-get-started/index.md +++ b/docs/en/05-get-started/index.md @@ -1,6 +1,6 @@ --- title: Get Started -description: This article describes how to install TDengine and test its performance. +description: This document describes how to install TDengine on various platforms. --- import GitHubSVG from './github.svg' diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 45bbaa2751a6ac6c04b4d6a7fcb00dd0fe03f638..913c24f1896e3c2af6614fc07e9319c6fb06783f 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -1,7 +1,7 @@ --- -sidebar_label: Connect title: Connect to TDengine -description: "How to establish connections to TDengine and how to install and use TDengine connectors." +sidebar_label: Connect +description: This document describes how to establish connections to TDengine and how to install and use TDengine connectors. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/07-develop/02-model/index.mdx b/docs/en/07-develop/02-model/index.mdx index 19a239805f1492eae0c463f5a942378b83ecf6b8..db5a259cfe6f8cc1fec6c588ec9b31888334ca06 100644 --- a/docs/en/07-develop/02-model/index.mdx +++ b/docs/en/07-develop/02-model/index.mdx @@ -1,5 +1,6 @@ --- title: Data Model +description: This document describes the data model of TDengine. --- The data model employed by TDengine is similar to that of a relational database. You have to create databases and tables. You must design the data model based on your own business and application requirements. You should design the [STable](/concept/#super-table-stable) (an abbreviation for super table) schema to fit your data. This chapter will explain the big picture without getting into syntactical details. diff --git a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx index f2168645ff9e59d60e88c85f86e890945b9f336c..3731882fb23677588e72ba5e9d39049af2dfd97d 100644 --- a/docs/en/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/en/07-develop/03-insert-data/01-sql-writing.mdx @@ -1,5 +1,6 @@ --- title: Insert Using SQL +description: This document describes how to insert data into TDengine using SQL. --- import Tabs from "@theme/Tabs"; @@ -29,25 +30,31 @@ Application programs can execute `INSERT` statement through connectors to insert The below SQL statement is used to insert one row into table "d1001". ```sql -INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31); +INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31); ``` +`ts1` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). + ### Insert Multiple Rows Multiple rows can be inserted in a single SQL statement. The example below inserts 2 rows into table "d1001". ```sql -INSERT INTO d1001 VALUES (1538548684000, 10.2, 220, 0.23) (1538548696650, 10.3, 218, 0.25); +INSERT INTO d1001 VALUES (ts2, 10.2, 220, 0.23) (ts2, 10.3, 218, 0.25); ``` +`ts1` and `ts2` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). + ### Insert into Multiple Tables Data can be inserted into multiple tables in the same SQL statement. The example below inserts 2 rows into table "d1001" and 1 row into table "d1002". ```sql -INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); +INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31) (ts2, 12.6, 218, 0.33) d1002 VALUES (ts3, 12.3, 221, 0.31); ``` +`ts1`, `ts2` and `ts3` is Unix timestamp, the timestamps which is larger than the difference between current time and KEEP in config is only allowed. For further detial, refer to [TDengine SQL insert timestamp section](/taos-sql/insert). + For more details about `INSERT` please refer to [INSERT](/taos-sql/insert). :::info diff --git a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx index ffb969a8a6d0bc61e23fee44f245a24236db5b1b..89ca10b669d094a156f55439429e329308e71935 100644 --- a/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx +++ b/docs/en/07-develop/03-insert-data/20-kafka-writting.mdx @@ -1,5 +1,6 @@ --- title: Write from Kafka +description: This document describes how to insert data into TDengine using Kafka. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx index 3c088602602301fafa824bc256f1f2caca128abd..c559883d26c7ef5750f3ee1f828912f8d800c3a4 100644 --- a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: InfluxDB Line Protocol title: InfluxDB Line Protocol +sidebar_label: InfluxDB Line Protocol +description: This document describes how to insert data into TDengine using the InfluxDB Line Protocol. --- import Tabs from "@theme/Tabs"; @@ -38,7 +39,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). - The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be created automatically. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. -- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) +- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, smlDataFormat is discarded since 3.0.3.0) ::: For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) diff --git a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 5d3f25dca96a0d41a4562aa067800607e8e04277..30bc3b87bf035f563e1a03508a90466e9288f89c 100644 --- a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: OpenTSDB Line Protocol title: OpenTSDB Line Protocol +sidebar_label: OpenTSDB Line Protocol +description: This document describes how to insert data into TDengine using the OpenTSDB Line Protocol. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index 7a3ac6bad3b203a9859e235e92d0865f7147d24c..e9db130241236e7224e5965c4c3c651ddab8cb00 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: OpenTSDB JSON Protocol title: OpenTSDB JSON Protocol +sidebar_label: OpenTSDB JSON Protocol +description: This document describes how to insert data into TDengine using the OpenTSDB JSON protocol. --- import Tabs from "@theme/Tabs"; @@ -47,7 +48,6 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http :::note - In JSON protocol, strings will be converted to NCHAR type and numeric values will be converted to double type. -- Only data in array format is accepted and so an array must be used even if there is only one row. - The child table name is created automatically in a rule to guarantee its uniqueness. But you can configure `smlChildTableName` in taos.cfg to specify a tag value as the table names if the tag value is unique globally. For example, if a tag is called `tname` and you set `smlChildTableName=tname` in taos.cfg, when you insert `st,tname=cpu1,t1=4 c1=3 1626006833639000000`, the child table `cpu1` will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: diff --git a/docs/en/07-develop/03-insert-data/60-high-volume.md b/docs/en/07-develop/03-insert-data/60-high-volume.md index 272a13881372b58a9a66c0d452cda54f9ee8c78c..d5afa8ef6ec120fa73854f5ca170504d7dd69b39 100644 --- a/docs/en/07-develop/03-insert-data/60-high-volume.md +++ b/docs/en/07-develop/03-insert-data/60-high-volume.md @@ -1,6 +1,7 @@ --- -sidebar_label: High Performance Writing title: High Performance Writing +sidebar_label: High Performance Writing +description: This document describes how to achieve high performance when writing data into TDengine. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/07-develop/03-insert-data/_py_kafka.mdx b/docs/en/07-develop/03-insert-data/_py_kafka.mdx index dc43a0d415275189220287278547b015865e8d90..c71821dad129fe1a573e6f607eb591f81f928ae1 100644 --- a/docs/en/07-develop/03-insert-data/_py_kafka.mdx +++ b/docs/en/07-develop/03-insert-data/_py_kafka.mdx @@ -53,8 +53,69 @@ for p in ps: In addition to python's built-in multithreading and multiprocessing library, we can also use the third-party library gunicorn. -### Examples +### examples + +
+kafka_example_perform + +`kafka_example_perform` is the entry point of the examples. + +```py +{{#include docs/examples/python/kafka_example_perform.py}} +``` +
+ +
+kafka_example_common + +`kafka_example_common` is the common code of the examples. ```py -{{#include docs/examples/python/kafka_example.py}} +{{#include docs/examples/python/kafka_example_common.py}} ``` +
+ +
+kafka_example_producer + +`kafka_example_producer` is `producer`, which is responsible for generating test data and sending it to kafka. + +```py +{{#include docs/examples/python/kafka_example_producer.py}} +``` +
+ +
+kafka_example_consumer + +`kafka_example_consumer` is `consumer`,which is responsible for consuming data from kafka and writing it to TDengine. + +```py +{{#include docs/examples/python/kafka_example_consumer.py}} +``` +
+ +### execute Python examples + +
+ execute Python examples + + 1. install and start up `kafka` + 2. install python3 and pip + 3. install `taospy` by pip + 4. install `kafka-python` by pip + 5. execute this example + + The entry point of this example is `kafka_example_perform.py`. For more information about usage, please use `--help` command. + + ``` + python3 kafka_example_perform.py --help + ``` + + For example, the following command is creating 100 sub-table and inserting 20000 data for each table and the kafka max poll is 100 and 1 thread and 1 process per thread. + + ``` + python3 kafka_example_perform.py -table-count=100 -table-items=20000 -max-poll=100 -threads=1 -processes=1 + ``` + +
diff --git a/docs/en/07-develop/03-insert-data/index.md b/docs/en/07-develop/03-insert-data/index.md index 1a71e719a56448e4b535632e570ce8a04d2282bb..15f8f4ee9e9b24e6b511e6d29505c7b99fea3ae8 100644 --- a/docs/en/07-develop/03-insert-data/index.md +++ b/docs/en/07-develop/03-insert-data/index.md @@ -1,5 +1,6 @@ --- title: Insert Data +description: This document describes how to insert data into TDengine. --- TDengine supports multiple protocols of inserting data, including SQL, InfluxDB Line protocol, OpenTSDB Telnet protocol, and OpenTSDB JSON protocol. Data can be inserted row by row, or in batches. Data from one or more collection points can be inserted simultaneously. Data can be inserted with multiple threads, and out of order data and historical data can be inserted as well. InfluxDB Line protocol, OpenTSDB Telnet protocol and OpenTSDB JSON protocol are the 3 kinds of schemaless insert protocols supported by TDengine. It's not necessary to create STables and tables in advance if using schemaless protocols, and the schemas can be adjusted automatically based on the data being inserted. diff --git a/docs/en/07-develop/04-query-data/index.mdx b/docs/en/07-develop/04-query-data/index.mdx index 38dc98d1ff262c7f8ec4951297e6f42e436682c8..7e167bb4f342167c486403259985976357e44496 100644 --- a/docs/en/07-develop/04-query-data/index.mdx +++ b/docs/en/07-develop/04-query-data/index.mdx @@ -1,6 +1,6 @@ --- title: Query Data -description: "This chapter introduces major query functionalities and how to perform sync and async query using connectors." +description: This document describes how to query data in TDengine and how to perform synchronous and asynchronous queries using connectors. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/07-develop/06-stream.md b/docs/en/07-develop/06-stream.md index 36f903ee9a4f2d210e63d0b79e702bc199f790ed..125173e60bae77bbbe782363ebd2103cf7e4b703 100644 --- a/docs/en/07-develop/06-stream.md +++ b/docs/en/07-develop/06-stream.md @@ -1,7 +1,7 @@ --- -sidebar_label: Stream Processing -description: "The TDengine stream processing engine combines data inserts, preprocessing, analytics, real-time computation, and alerting into a single component." title: Stream Processing +sidebar_label: Stream Processing +description: This document describes the stream processing component of TDengine. --- Raw time-series data is often cleaned and preprocessed before being permanently stored in a database. In a traditional time-series solution, this generally requires the deployment of stream processing systems such as Kafka or Flink. However, the complexity of such systems increases the cost of development and maintenance. diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 17b3f5caa062eaacb4216b7153e899040e702cc1..c85109d3c5b5e2ea4d02311dd930b2bb2fe2040c 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -94,22 +94,21 @@ void close() throws SQLException; ```python -class TaosConsumer(): - def __init__(self, *topics, **configs) +class Consumer: + def subscribe(self, topics): + pass - def __iter__(self) + def unsubscribe(self): + pass - def __next__(self) + def poll(self, timeout: float = 1.0): + pass - def sync_next(self) - - def subscription(self) + def close(self): + pass - def unsubscribe(self) - - def close(self) - - def __del__(self) + def commit(self, message): + pass ``` @@ -117,19 +116,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) - -func (c *Consumer) Close() error +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// rebalanceCb is reserved for compatibility purpose +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// tmq.TopicPartition is reserved for compatibility purpose +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -220,7 +222,7 @@ A database including one supertable and two subtables is created as follows: ```sql DROP DATABASE IF EXISTS tmqdb; CREATE DATABASE tmqdb; -CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16) TAGS(t1 INT, t3 VARCHAR(16)); +CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00'); @@ -357,50 +359,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -422,23 +394,31 @@ let mut consumer = tmq.build()?; +```python +from taos.tmq import Consumer + +# Syntax: `consumer = Consumer(configs)` +# +# Example: +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) +``` + Python programs use the following parameters: -| Parameter | Type | Description | Remarks | -| :----------------------------: | :----: | -------------------------------------------------------- | ------------------------------------------- | -| `td_connect_ip` | string | Used in establishing a connection; same as `taos_connect` | | -| `td_connect_user` | string | Used in establishing a connection; same as `taos_connect` | | -| `td_connect_pass` | string | Used in establishing a connection; same as `taos_connect` | | -| `td_connect_port` | string | Used in establishing a connection; same as `taos_connect` | | -| `group_id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192. | -| `client_id` | string | Client ID | Maximum length: 192. | -| `auto_offset_reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | -| `enable_auto_commit` | string | Commit automatically | Specify `true` or `false`. | -| `auto_commit_interval_ms` | string | Interval for automatic commits, in milliseconds | -| `enable_heartbeat_background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false`. | -| `experimental_snapshot_enable` | string | Specify whether to consume messages from the WAL or from TSBS | Specify `true` or `false`. | -| `msg_with_table_name` | string | Specify whether to deserialize table names from messages | Specify `true` or `false`. -| `timeout` | int | Consumer pull timeout | | +| Parameter | Type | Description | Remarks | +|:---------:|:----:|:-----------:|:-------:| +| `td.connect.ip` | string | Used in establishing a connection|| +| `td.connect.user` | string | Used in establishing a connection|| +| `td.connect.pass` | string | Used in establishing a connection|| +| `td.connect.port` | string | Used in establishing a connection|| +| `group.id` | string | Consumer group ID; consumers with the same ID are in the same group | **Required**. Maximum length: 192 | +| `client.id` | string | Client ID | Maximum length: 192 | +| `msg.with.table.name` | string | Specify whether to deserialize table names from messages | pecify `true` or `false` | +| `enable.auto.commit` | string | Commit automatically | pecify `true` or `false` | +| `auto.commit.interval.ms` | string | Interval for automatic commits, in milliseconds | | +| `auto.offset.reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | +| `experimental.snapshot.enable` | string | Specify whether to consume messages from the WAL or from TSDB | Specify `true` or `false` | +| `enable.heartbeat.background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false` | @@ -523,11 +503,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -545,7 +521,7 @@ consumer.subscribe(["tmq_meters"]).await?; ```python -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') +consumer.subscribe(['topic1', 'topic2']) ``` @@ -611,13 +587,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -660,9 +640,17 @@ for { ```python -for msg in consumer: - for row in msg: - print(row) +while True: + res = consumer.poll(100) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) ``` @@ -729,7 +717,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/en/07-develop/08-cache.md b/docs/en/07-develop/08-cache.md index 82a4787016f608f8e32e89b1747443b7cd164551..6a6ca3e5947b6a3233f90e5e0a01417e6be3b19d 100644 --- a/docs/en/07-develop/08-cache.md +++ b/docs/en/07-develop/08-cache.md @@ -1,7 +1,7 @@ --- -sidebar_label: Caching title: Caching -description: "This document describes the caching component of TDengine." +sidebar_label: Caching +description: This document describes the caching component of TDengine. --- TDengine uses various kinds of caching techniques to efficiently write and query data. This document describes the caching component of TDengine. diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index 699b3ebe5f7cddaa6f56cb8027c784a5821a7a0d..553a7b932b013a1f4d2b5e4e291f71c28b64039f 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -1,7 +1,7 @@ --- -sidebar_label: UDF title: User-Defined Functions (UDF) -description: "You can define your own scalar and aggregate functions to expand the query capabilities of TDengine." +sidebar_label: UDF +description: This document describes how to create user-defined functions (UDF), your own scalar and aggregate functions that can expand the query capabilities of TDengine. --- The built-in functions of TDengine may not be sufficient for the use cases of every application. In this case, you can define custom functions for use in TDengine queries. These are known as user-defined functions (UDF). A user-defined function takes one column of data or the result of a subquery as its input. diff --git a/docs/en/07-develop/_sub_java.mdx b/docs/en/07-develop/_sub_java.mdx index d14b5fd6095dd90f89dd2c2e828858585cfddff9..965161651c188bedc01c58a11d127035e11b885d 100644 --- a/docs/en/07-develop/_sub_java.mdx +++ b/docs/en/07-develop/_sub_java.mdx @@ -1,11 +1,31 @@ -```java -{{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}} -{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} -{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} -``` -```java -{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} -``` -```java -{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} -``` \ No newline at end of file +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}} + ``` + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} + ``` + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} + ``` + + + + + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java}} + ``` + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} + ``` + ```java + {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} + ``` + + + diff --git a/docs/en/07-develop/index.md b/docs/en/07-develop/index.md index 34649d32a2f758fda2bd79afbbd8f3427b714353..8f80b82b970c576945eee6f972bd901d910614a9 100644 --- a/docs/en/07-develop/index.md +++ b/docs/en/07-develop/index.md @@ -1,5 +1,6 @@ --- title: Developer Guide +description: This document describes how to use the various components of TDengine from a developer's perspective. --- Before creating an application to process time-series data with TDengine, consider the following: diff --git a/docs/en/10-deployment/01-deploy.md b/docs/en/10-deployment/01-deploy.md index 5dfcd3108d8b10cf24cdd5c852c4225ced0f16b2..da00e21a7ec7c7a34c0a3ae60b2928bd3b036cc4 100644 --- a/docs/en/10-deployment/01-deploy.md +++ b/docs/en/10-deployment/01-deploy.md @@ -1,6 +1,7 @@ --- -sidebar_label: Manual Deployment title: Manual Deployment and Management +sidebar_label: Manual Deployment +description: This document describes how to deploy TDengine on a server. --- ## Prerequisites diff --git a/docs/en/10-deployment/03-k8s.md b/docs/en/10-deployment/03-k8s.md index b0aa6777130864404e97dc332cf0e5ce830bf8ed..49e61caafce6414a10c4bc1937ce13394a7da9eb 100644 --- a/docs/en/10-deployment/03-k8s.md +++ b/docs/en/10-deployment/03-k8s.md @@ -1,6 +1,7 @@ --- -sidebar_label: Kubernetes title: Deploying a TDengine Cluster in Kubernetes +sidebar_label: Kubernetes +description: This document describes how to deploy TDengine on Kubernetes. --- TDengine is a cloud-native time-series database that can be deployed on Kubernetes. This document gives a step-by-step description of how you can use YAML files to create a TDengine cluster and introduces common operations for TDengine in a Kubernetes environment. diff --git a/docs/en/10-deployment/05-helm.md b/docs/en/10-deployment/05-helm.md index a4fa68100078efe85fff5e1b078ebd07e5337d5a..aa617176693a576c8dbd37b83a70b1be3b983ef0 100644 --- a/docs/en/10-deployment/05-helm.md +++ b/docs/en/10-deployment/05-helm.md @@ -1,6 +1,7 @@ --- -sidebar_label: Helm title: Use Helm to deploy TDengine +sidebar_label: Helm +description: This document describes how to deploy TDengine on Kubernetes by using Helm. --- Helm is a package manager for Kubernetes that can provide more capabilities in deploying on Kubernetes. @@ -22,7 +23,7 @@ Helm uses the kubectl and kubeconfig configurations to perform Kubernetes operat To use TDengine Chart, download it from GitHub: ```bash -wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.2.tgz ``` @@ -38,7 +39,7 @@ With minikube, the default value is standard. Use Helm commands to install TDengine: ```bash -helm install tdengine tdengine-3.0.0.tgz \ +helm install tdengine tdengine-3.0.2.tgz \ --set storage.className= ``` @@ -46,7 +47,7 @@ helm install tdengine tdengine-3.0.0.tgz \ You can configure a small storage size in minikube to ensure that your deployment does not exceed your available disk space. ```bash -helm install tdengine tdengine-3.0.0.tgz \ +helm install tdengine tdengine-3.0.2.tgz \ --set storage.className=standard \ --set storage.dataSize=2Gi \ --set storage.logSize=10Mi @@ -83,14 +84,14 @@ You can configure custom parameters in TDengine with the `values.yaml` file. Run the `helm show values` command to see all parameters supported by TDengine Chart. ```bash -helm show values tdengine-3.0.0.tgz +helm show values tdengine-3.0.2.tgz ``` Save the output of this command as `values.yaml`. Then you can modify this file with your desired values and use it to deploy a TDengine cluster: ```bash -helm install tdengine tdengine-3.0.0.tgz -f values.yaml +helm install tdengine tdengine-3.0.2.tgz -f values.yaml ``` @@ -107,7 +108,7 @@ image: prefix: tdengine/tdengine #pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. -# tag: "3.0.0.0" +# tag: "3.0.2.0" service: # ClusterIP is the default service type, use NodeIP only if you know what you are doing. @@ -155,15 +156,15 @@ clusterDomainSuffix: "" # See the [Configuration Variables](../../reference/config) # # Note: -# 1. firstEp/secondEp: should not be setted here, it's auto generated at scale-up. -# 2. serverPort: should not be setted, we'll use the default 6030 in many places. -# 3. fqdn: will be auto generated in kubenetes, user should not care about it. +# 1. firstEp/secondEp: should not be set here, it's auto generated at scale-up. +# 2. serverPort: should not be set, we'll use the default 6030 in many places. +# 3. fqdn: will be auto generated in kubernetes, user should not care about it. # 4. role: currently role is not supported - every node is able to be mnode and vnode. # # Btw, keep quotes "" around the value like below, even the value will be number or not. taoscfg: # Starts as cluster or not, must be 0 or 1. - # 0: all pods will start as a seperate TDengine server + # 0: all pods will start as a separate TDengine server # 1: pods will start as TDengine server cluster. [default] CLUSTER: "1" diff --git a/docs/en/10-deployment/index.md b/docs/en/10-deployment/index.md index 7054a33e4a40222ed5eb9a15837990e3e7a81cff..865fbc2da57bb985dabc03a743c5e0f7186b98be 100644 --- a/docs/en/10-deployment/index.md +++ b/docs/en/10-deployment/index.md @@ -1,5 +1,6 @@ --- title: Deployment +description: This document describes how to deploy a TDengine cluster on a server, on Kubernetes, and by using Helm. --- TDengine has a native distributed design and provides the ability to scale out. A few nodes can form a TDengine cluster. If you need higher processing power, you just need to add more nodes into the cluster. TDengine uses virtual node technology to virtualize a node into multiple virtual nodes to achieve load balancing. At the same time, TDengine can group virtual nodes on different nodes into virtual node groups, and use the replication mechanism to ensure the high availability of the system. The cluster feature of TDengine is completely open source. diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 60046629a4c6f89ccfe9b20adcbb2fdba2ffb261..641fd3cbb7c4e2937cc1756a41a522f6eb4b51aa 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -1,7 +1,7 @@ --- -sidebar_label: Data Types title: Data Types -description: 'TDengine supports a variety of data types including timestamp, float, JSON and many others.' +sidebar_label: Data Types +description: This document describes the data types that TDengine supports. --- ## Timestamp diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index a12406fe4353d437c5df9755f8f8e68b0f24282f..5e8c9aaa929032441459a55015fec8d591d32704 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -1,7 +1,7 @@ --- -sidebar_label: Database title: Database -description: "create and drop database, show or change database parameters" +sidebar_label: Database +description: This document describes how to create and perform operations on databases. --- ## Create a Database @@ -27,10 +27,13 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value + | TABLE_PREFIX value + | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -55,15 +58,12 @@ database_option: { - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk. - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096. - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100. -- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. +- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. The Enterprise Edition supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; the Community Edition does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP). - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB. - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384. - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms. - REPLICA: specifies the number of replicas that are made of the database. Enter 1 or 3. The default value is 1. The value of the REPLICA parameter cannot exceed the number of dnodes in the cluster. - RETENTIONS: specifies the retention period for data aggregated at various intervals. For example, RETENTIONS 15s:7d,1m:21d,15m:50d indicates that data aggregated every 15 seconds is retained for 7 days, data aggregated every 1 minute is retained for 21 days, and data aggregated every 15 minutes is retained for 50 days. You must enter three aggregation intervals and corresponding retention periods. -- STRICT: specifies whether strong data consistency is enabled. The default value is off. - - on: Strong consistency is enabled and implemented through the Raft consensus algorithm. In this mode, an operation is considered successful once it is confirmed by half of the nodes in the cluster. - - off: Strong consistency is disabled. In this mode, an operation is considered successful when it is initiated by the local node. - WAL_LEVEL: specifies whether fsync is enabled. The default value is 1. - 1: WAL is enabled but fsync is disabled. - 2: WAL and fsync are both enabled. @@ -71,6 +71,10 @@ database_option: { - SINGLE_STABLE: specifies whether the database can contain more than one supertable. - 0: The database can contain multiple supertables. - 1: The database can contain only one supertable. +- STT_TRIGGER: specifies the number of file merges triggered by flushed files. The default is 8, ranging from 1 to 16. For high-frequency scenarios with few tables, it is recommended to use the default configuration or a smaller value for this parameter; For multi-table low-frequency scenarios, it is recommended to configure this parameter with a larger value. +- TABLE_PREFIX:The prefix length in the table name that is ignored when distributing table to vnode based on table name. +- TABLE_SUFFIX:The suffix length in the table name that is ignored when distributing table to vnode based on table name. +- TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. - WAL_RETENTION_PERIOD: specifies the time after which WAL files are deleted. This parameter is used for data subscription. Enter a time in seconds. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is 4 days. - WAL_RETENTION_SIZE: specifies the size at which WAL files are deleted. This parameter is used for data subscription. Enter a size in KB. The default value of single copy is 0. A value of 0 indicates that each WAL file is deleted immediately after its contents are written to disk. -1: WAL files are never deleted. The default value of multiple copy is -1. - WAL_ROLL_PERIOD: specifies the time after which WAL files are rotated. After this period elapses, a new WAL file is created. The default value of single copy is 0. A value of 0 indicates that a new WAL file is created only after the previous WAL file was written to disk. The default values of multiple copy is 1 day. @@ -112,12 +116,32 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value } ``` +### ALTER CACHESIZE + +The command of changing database configuration parameters is easy to use, but it's hard to determine whether a parameter is proper or not. In this section we will describe how to determine whether cachesize is big enough. + +1. How to check cachesize? + +You can use `select * from information_schema.ins_databases;` to get the value of cachesize. + +2. How to check cacheload? + +You can use `show .vgroups;` to check the value of cacheload. + +3. Determine whether cachesize is big engough + +If the value of `cacheload` is very close to the value of `cachesize`, then it's very probably that `cachesize` is too small. If the value of `cacheload` is much smaller than the value of `cachesize`, then `cachesize` is big enough. You can use this simple principle to determine. Depending on how much memory is available in your system, you can choose to double `cachesize` or incrase it by even 5 or more times. + :::note Other parameters cannot be modified after the database has been created. @@ -154,3 +178,19 @@ TRIM DATABASE db_name; ``` The preceding SQL statement deletes data that has expired and orders the remaining data in accordance with the storage configuration. + +## Redistribute Vgroup + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +Adjust the distribution of vnodes in the vgroup according to the given list of dnodes. + +## Balance Vgroup + +```sql +BALANCE VGROUP +``` + +Automatically adjusts the distribution of vnodes in all vgroups of the cluster, which is equivalent to load balancing the data of the cluster at the vnode level. diff --git a/docs/en/12-taos-sql/03-table.md b/docs/en/12-taos-sql/03-table.md index 6d12b8c73056d049997a244ed0ab788742094d65..f61d1f51471bf042aea79023bff4f74c43831eb8 100644 --- a/docs/en/12-taos-sql/03-table.md +++ b/docs/en/12-taos-sql/03-table.md @@ -1,5 +1,6 @@ --- title: Table +description: This document describes how to create and perform operations on standard tables and subtables. --- ## Create Table diff --git a/docs/en/12-taos-sql/04-stable.md b/docs/en/12-taos-sql/04-stable.md index 8a7c713f8cad6d61c8b098583cc4e282b00d48cf..5b316d0d2460b497655e7f29acb897301b5b4769 100644 --- a/docs/en/12-taos-sql/04-stable.md +++ b/docs/en/12-taos-sql/04-stable.md @@ -1,6 +1,7 @@ --- -sidebar_label: Supertable title: Supertable +sidebar_label: Supertable +description: This document describes how to create and perform operations on supertables. --- ## Create a Supertable diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 9141211db5239cb6a035f4a01a831adf9f50aa77..c22357abfa1c9e2a4dece880c407cb4bbfdaa055 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -1,6 +1,7 @@ --- -sidebar_label: Insert title: Insert +sidebar_label: Insert +description: This document describes how to insert data into TDengine. --- ## Syntax @@ -27,7 +28,7 @@ INSERT INTO tb_name [(field1_name, ...)] subquery 2. The precision of a timestamp depends on its format. The precision configured for the database affects only timestamps that are inserted as long integers (UNIX time). Timestamps inserted as date and time strings are not affected. As an example, the timestamp 2021-07-13 16:16:48 is equivalent to 1626164208 in UNIX time. This UNIX time is modified to 1626164208000 for databases with millisecond precision, 1626164208000000 for databases with microsecond precision, and 1626164208000000000 for databases with nanosecond precision. 3. If you want to insert multiple rows simultaneously, do not use the NOW function in the timestamp. Using the NOW function in this situation will cause multiple rows to have the same timestamp and prevent them from being stored correctly. This is because the NOW function obtains the current time on the client, and multiple instances of NOW in a single statement will return the same time. - The earliest timestamp that you can use when inserting data is equal to the current time on the server minus the value of the KEEP parameter. The latest timestamp that you can use when inserting data is equal to the current time on the server plus the value of the DURATION parameter. You can configure the KEEP and DURATION parameters when you create a database. The default values are 3650 days for the KEEP parameter and 10 days for the DURATION parameter. + The earliest timestamp that you can use when inserting data is equal to the current time on the server minus the value of the KEEP parameter (You can configure the KEEP parameter when you create a database and the default value is 3650 days). The latest timestamp you can use when inserting data depends on the PRECISION parameter (You can configure the PRECISION parameter when you create a database, ms means milliseconds, us means microseconds, ns means nanoseconds, and the default value is milliseconds). If the timestamp precision is milliseconds or microseconds, the latest timestamp is the Unix epoch (January 1st, 1970 at 00:00:00.000 UTC) plus 1000 years, that is, January 1st, 2970 at 00:00:00.000 UTC; If the timestamp precision is nanoseconds, the latest timestamp is the Unix epoch plus 292 years, that is, January 1st, 2262 at 00:00:00.000000000 UTC. **Syntax** diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 485779f55f292f04ecebfc75dd82ddfa240d98bf..3099e3a541861ad236135a5bef1255db7d4394b9 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -1,6 +1,7 @@ --- -sidebar_label: Select title: Select +sidebar_label: Select +description: This document describes how to query data in TDengine. --- ## Syntax @@ -354,9 +355,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN -TDengine supports natural joins between supertables, between standard tables, and between subqueries. The difference between natural joins and inner joins is that natural joins require that the fields being joined in the supertables or standard tables must have the same name. Data or tag columns must be joined with the equivalent column in another table. +TDengine supports the `INTER JOIN` based on the timestamp primary key, that is, the `JOIN` condition must contain the timestamp primary key. As long as the requirement of timestamp-based primary key is met, `INTER JOIN` can be made between normal tables, sub-tables, super tables and sub-queries at will, and there is no limit on the number of tables. -For standard tables, only the timestamp (primary key) can be used in join operations. For example: +For standard tables: ```sql SELECT * @@ -364,7 +365,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -For supertables, tags as well as timestamps can be used in join operations. For example: +For supertables: ```sql SELECT * @@ -372,20 +373,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -Similarly, join operations can be performed on the result sets of multiple subqueries. - -:::note - -The following restriction apply to JOIN statements: +For sub-table and super table: -- The number of tables or supertables in a single join operation cannot exceed 10. -- `FILL` cannot be used in a JOIN statement. -- Arithmetic operations cannot be performed on the result sets of join operation. -- `GROUP BY` is not allowed on a segment of the tables that participate in a join operation. -- `OR` cannot be used in the conditions for join operation -- Join operation can be performed only on tags or timestamps. You cannot perform a join operation on data columns. +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +Similarly, join operations can be performed on the result sets of multiple subqueries. ## Nested Query diff --git a/docs/en/12-taos-sql/08-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx index 999c467ad05b6d3e349e322141acfb02a49de5ff..f91a89a7ebe86a7699f5c9a3fd6d3bca200c1445 100644 --- a/docs/en/12-taos-sql/08-delete-data.mdx +++ b/docs/en/12-taos-sql/08-delete-data.mdx @@ -1,7 +1,7 @@ --- -sidebar_label: Delete Data -description: "Delete data from table or Stable" title: Delete Data +sidebar_label: Delete Data +description: This document describes how to delete data from TDengine. --- TDengine provides the functionality of deleting data from a table or STable according to specified time range, it can be used to cleanup abnormal data generated due to device failure. diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 8ab5176aab0a3cd9a2dc4395febbf6ddc8f595f3..b32d2af5bb0787c748dc84a922700c2db43c1d00 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -1,6 +1,7 @@ --- -sidebar_label: Functions title: Functions +sidebar_label: Functions +description: This document describes the standard SQL functions available in TDengine. toc_max_heading_level: 4 --- @@ -795,19 +796,23 @@ HISTOGRAM(expr,bin_type, bin_description, normalized) ### PERCENTILE ```sql -PERCENTILE(expr, p) +PERCENTILE(expr, p [, p1] ...) ``` **Description**: The value whose rank in a specific column matches the specified percentage. If such a value matching the specified percentage doesn't exist in the column, an interpolation value will be returned. -**Return value type**: DOUBLE +**Return value type**: This function takes 2 minumum and 11 maximum parameters, and it can simultaneously return 10 percentiles at most. If 2 parameters are given, a single percentile is returned and the value type is DOUBLE. + If more than 2 parameters are given, the return value type is a VARCHAR string, the format of which is a JSON ARRAY containing all return values. **Applicable column types**: Numeric **Applicable table types**: table only -**More explanations**: _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX. +**More explanations**: +- _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX. +- When calculating multiple percentiles of a specific column, a single PERCENTILE function with multiple parameters is adviced, as this can largely reduce the query response time. + For example, using SELECT percentile(col, 90, 95, 99) FROM table will perform better than SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table. ## Selection Functions @@ -876,7 +881,8 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY(time_unit)`. Starting from timestamp1, one interpolation is performed for every time interval specified `time_unit` parameter. The parameter `time_unit` must be an integer, with no quotes, with a time unit of: a(millisecond)), s(second), m(minute), h(hour), d(day), or w(week). For example, `EVERY(500a)` will interpolate every 500 milliseconds. - Interpolation is performed based on `FILL` parameter. For more information about FILL clause, see [FILL Clause](../distinguished/#fill-clause). - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. -- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). +- Pseudocolumn `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.2.0). +- Pseudocolumn `_isfilled` can be used along with `INTERP` to indicate whether the results are original records or data points generated by interpolation algorithm(support after version 3.0.3.0). ### LAST diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 0763e85a53bc6021a2fe28d7df20f627d1b3fbe7..536fd8ffc3968996b8e31b5a2864fd730a7624a7 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -1,6 +1,7 @@ --- -sidebar_label: Time-Series Extensions title: Time-Series Extensions +sidebar_label: Time-Series Extensions +description: This document describes the extended functions specific to time-series data processing available in TDengine. --- As a purpose-built database for storing and processing time-series data, TDengine provides time-series-specific extensions to standard SQL. diff --git a/docs/en/12-taos-sql/13-tmq.md b/docs/en/12-taos-sql/13-tmq.md index befab4f4f01e595564e93ffcfbb0723e13294af0..1a805c76fb8e51ad22dd4e8b090860a6e1730a24 100644 --- a/docs/en/12-taos-sql/13-tmq.md +++ b/docs/en/12-taos-sql/13-tmq.md @@ -1,6 +1,7 @@ --- -sidebar_label: Data Subscription title: Data Subscription +sidebar_label: Data Subscription +description: This document describes the SQL statements related to the data subscription component of TDengine. --- The information in this document is related to the TDengine data subscription feature. diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index c47d2da0ebf8b8d3ecc2e51fad659ceb423ad46f..b8f6c3a163e1c2fa37e437725652eabb1a71dbe6 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -1,6 +1,7 @@ --- -sidebar_label: Stream Processing title: Stream Processing +sidebar_label: Stream Processing +description: This document describes the SQL statements related to the stream processing component of TDengine. --- Raw time-series data is often cleaned and preprocessed before being permanently stored in a database. Stream processing components like Kafka, Flink, and Spark are often deployed alongside a time-series database to handle these operations, increasing system complexity and maintenance costs. @@ -108,7 +109,7 @@ SHOW STREAMS; When you create a stream, you can use the TRIGGER parameter to specify triggering conditions for it. -For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering: +For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering,the default value is AT_ONCE: 1. AT_ONCE: triggers on write diff --git a/docs/en/12-taos-sql/16-operators.md b/docs/en/12-taos-sql/16-operators.md index c426e2879342e430c61c4f8133aa9f8186888941..32ad4e7075b6a510cb537016effb6064e6c51794 100644 --- a/docs/en/12-taos-sql/16-operators.md +++ b/docs/en/12-taos-sql/16-operators.md @@ -1,6 +1,7 @@ --- -sidebar_label: Operators title: Operators +sidebar_label: Operators +description: This document describes the SQL operators available in TDengine. --- ## Arithmetic Operators diff --git a/docs/en/12-taos-sql/17-json.md b/docs/en/12-taos-sql/17-json.md index 77f774303316b466a15226f548f84da69be8f92d..b2494e0cc1a1829d0389b2cf634e5d8575817f57 100644 --- a/docs/en/12-taos-sql/17-json.md +++ b/docs/en/12-taos-sql/17-json.md @@ -1,6 +1,7 @@ --- -sidebar_label: JSON Type title: JSON Type +sidebar_label: JSON Type +description: This document describes the JSON data type in TDengine. --- diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md index a2ae40de98be677e599e83a634952a39faeaafbf..85e4610e44f1a6f93e9727c32f8333cddf07ecc7 100644 --- a/docs/en/12-taos-sql/18-escape.md +++ b/docs/en/12-taos-sql/18-escape.md @@ -1,5 +1,6 @@ --- title: Escape Characters +description: This document describes the usage of escape characters in TDengine. --- ## Escape Characters diff --git a/docs/en/12-taos-sql/19-limit.md b/docs/en/12-taos-sql/19-limit.md index f00ec90f5755d3d004a35b210f93560c6f719536..654fae7560d3c24df1353b87952a3868ca307418 100644 --- a/docs/en/12-taos-sql/19-limit.md +++ b/docs/en/12-taos-sql/19-limit.md @@ -1,6 +1,7 @@ --- -sidebar_label: Name and Size Limits title: Name and Size Limits +sidebar_label: Name and Size Limits +description: This document describes the name and size limits in TDengine. --- ## Naming Rules diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index 4b479b866b77e1e354d20376ccb869755af76d00..a2191c87ee17444d5341b8e97b33a954be84dc98 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -1,6 +1,7 @@ --- -sidebar_label: Reserved Keywords title: Reserved Keywords +sidebar_label: Reserved Keywords +description: This document describes the reserved keywords in TDengine that cannot be used in object names. --- ## Keyword List @@ -17,6 +18,7 @@ The following list shows all reserved keywords: - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/en/12-taos-sql/21-node.md b/docs/en/12-taos-sql/21-node.md index a0d49ab208448fa9dd90ef63deb235ef051b0e19..8a5069e66f5111c3c362829cfd654f16a80b60c7 100644 --- a/docs/en/12-taos-sql/21-node.md +++ b/docs/en/12-taos-sql/21-node.md @@ -1,6 +1,7 @@ --- -sidebar_label: Cluster title: Cluster +sidebar_label: Cluster +description: This document describes the SQL statements related to cluster management in TDengine. --- The physical entities that form TDengine clusters are known as data nodes (dnodes). Each dnode is a process running on the operating system of the physical machine. Dnodes can contain virtual nodes (vnodes), which store time-series data. Virtual nodes are formed into vgroups, which have 1 or 3 vnodes depending on the replica setting. If you want to enable replication on your cluster, it must contain at least three nodes. Dnodes can also contain management nodes (mnodes). Each cluster has up to three mnodes. Finally, dnodes can contain query nodes (qnodes), which compute time-series data, thus separating compute from storage. A single dnode can contain a vnode, qnode, and mnode. diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index 1cd759742a8714274712343a22cbf723cd9c3212..d2bc72f047a7e18c120a39e426caea70fd711018 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -1,6 +1,7 @@ --- -sidebar_label: Metadata title: Information_Schema Database +sidebar_label: Metadata +description: This document describes how to use the INFORMATION_SCHEMA database in TDengine. --- TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide access to database metadata, system information, and status information. This information includes database names, table names, and currently running SQL statements. All information related to TDengine maintenance is stored in this database. It contains several read-only tables. These tables are more accurately described as views, and they do not correspond to specific files. You can query these tables but cannot write data to them. The INFORMATION_SCHEMA database is intended to provide a unified method for SHOW commands to access data. However, using SELECT ... FROM INFORMATION_SCHEMA.tablename offers several advantages over SHOW commands: diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md index 29cf3af6abfbbc06e42ae99c78f35f33a3c7c30a..fc369ec663cf9130bd7d1af650f1f47a1054fad6 100644 --- a/docs/en/12-taos-sql/23-perf.md +++ b/docs/en/12-taos-sql/23-perf.md @@ -1,6 +1,7 @@ --- -sidebar_label: Statistics title: Performance_Schema Database +sidebar_label: Statistics +description: This document describes how to use the PERFORMANCE_SCHEMA database in TDengine. --- TDengine includes a built-in database named `PERFORMANCE_SCHEMA` to provide access to database performance statistics. This document introduces the tables of PERFORMANCE_SCHEMA and their structure. diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 1bda4a118d796765f2dab6f5fdeae27a58cdd232..dc1db956a0db1eead3a6c729e51c140b5be30c37 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -1,6 +1,7 @@ --- -sidebar_label: SHOW Statement title: SHOW Statement for Metadata +sidebar_label: SHOW Statement +description: This document describes how to use the SHOW statement in TDengine. --- `SHOW` command can be used to get brief system information. To get details about metatadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`. @@ -178,6 +179,139 @@ SHOW TABLE DISTRIBUTED table_name; Shows how table data is distributed. +Examples: Below is an example of this command to display the block distribution of table `d0` in detailed format. + +```sql +show table distributed d0\G; +``` + +
+ Show Example +

+*************************** 1.row ***************************
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+
+Total_Blocks :  Table `d0` contains total 5 blocks
+
+Total_Size:  The total size of all the data blocks in table `d0` is 93.65 KB 
+
+Average_size:  The average size of each block is 18.73 KB
+
+Compression_Ratio: The data compression rate is 23.98%
+ 
+*************************** 2.row ***************************
+_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
+
+Total_Rows: Table `d0` contains 20,000 rows
+
+Inmem_Rows: The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows
+
+MinRows:  The minimum number of rows in a block is 3,616 
+
+MaxRows: The maximum number of rows in a block is 4,096B
+
+Average_Rows: The average number of rows in a block is 4,000
+
+*************************** 3.row ***************************
+_block_dist: Total_Tables=[1] Total_Files=[2]
+
+Total_Tables:  The number of child tables, 1 in this example
+
+Total_Files:   The number of files storing the table's data, 2 in this example
+
+*************************** 4.row ***************************
+
+_block_dist: --------------------------------------------------------------------------------
+
+*************************** 5.row ***************************
+
+_block_dist: 0100 |
+
+*************************** 6.row ***************************
+
+_block_dist: 0299 |
+
+*************************** 7.row ***************************
+
+_block_dist: 0498 |
+
+*************************** 8.row ***************************
+
+_block_dist: 0697 |
+
+*************************** 9.row ***************************
+
+_block_dist: 0896 |
+
+*************************** 10.row ***************************
+
+_block_dist: 1095 |
+
+*************************** 11.row ***************************
+
+_block_dist: 1294 |
+
+*************************** 12.row ***************************
+
+_block_dist: 1493 |
+
+*************************** 13.row ***************************
+
+_block_dist: 1692 |
+
+*************************** 14.row ***************************
+
+_block_dist: 1891 |
+
+*************************** 15.row ***************************
+
+_block_dist: 2090 |
+
+*************************** 16.row ***************************
+
+_block_dist: 2289 |
+
+*************************** 17.row ***************************
+
+_block_dist: 2488 |
+
+*************************** 18.row ***************************
+
+_block_dist: 2687 |
+
+*************************** 19.row ***************************
+
+_block_dist: 2886 |
+
+*************************** 20.row ***************************
+
+_block_dist: 3085 |
+
+*************************** 21.row ***************************
+
+_block_dist: 3284 |
+
+*************************** 22.row ***************************
+
+_block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
+*************************** 23.row ***************************
+
+_block_dist: 3682 |
+
+*************************** 24.row ***************************
+
+_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
+Query OK, 24 row(s) in set (0.002444s)
+
+
+
+ + The above show the block distribution percentage according to the number of rows in each block. In the above example, we can get below information: + - `_block_dist: 3483 ||||||||||||||||| 1 (20.00%)` means there is one block whose rows is between 3,483 and 3,681. + - `_block_dist: 3881 ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| 4 (80.00%)` means there are 4 blocks whose rows is between 3,881 and 4,096. - The number of blocks whose rows fall in other range is zero. + ## SHOW TAGS ```sql @@ -230,7 +364,7 @@ Shows information about all vgroups in the system or about the vgroups for a spe ## SHOW VNODES ```sql -SHOW VNODES [dnode_name]; +SHOW VNODES {dnode_id | dnode_endpoint}; ``` Shows information about all vnodes in the system or about the vnodes for a specified dnode. diff --git a/docs/en/12-taos-sql/25-grant.md b/docs/en/12-taos-sql/25-grant.md index f895567c621f123778c8a711a68357a889ca0098..8b4c4393523c90595c7dd5461fc2abc73fa44c93 100644 --- a/docs/en/12-taos-sql/25-grant.md +++ b/docs/en/12-taos-sql/25-grant.md @@ -1,7 +1,7 @@ --- -sidebar_label: Access Control title: User and Access Control -description: Manage user and user's permission +sidebar_label: Access Control +description: This document describes how to manage users and permissions in TDengine. --- This document describes how to manage permissions in TDengine. diff --git a/docs/en/12-taos-sql/26-udf.md b/docs/en/12-taos-sql/26-udf.md index 977f3bcc0844ad63e4d2fe0f28d9ca8f29c5cbd6..cb64873705c471fa23288b604df5c03fcade2ba0 100644 --- a/docs/en/12-taos-sql/26-udf.md +++ b/docs/en/12-taos-sql/26-udf.md @@ -1,6 +1,7 @@ --- -sidebar_label: User-Defined Functions title: User-Defined Functions (UDF) +sidebar_label: User-Defined Functions +description: This document describes the SQL statements related to user-defined functions (UDF) in TDengine. --- You can create user-defined functions and import them into TDengine. @@ -40,7 +41,7 @@ CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ ```sql CREATE AGGREGATE FUNCTION l2norm AS "/home/taos/udf_example/libl2norm.so" OUTPUTTYPE DOUBLE bufsize 8; ``` -For more information about user-defined functions, see [User-Defined Functions](../../develop/udf). +For more information about user-defined functions, see [User-Defined Functions](/develop/udf). ## Manage UDF diff --git a/docs/en/12-taos-sql/27-index.md b/docs/en/12-taos-sql/27-index.md index 7d09bc43ab06932b82019923d4a8fda48cd99c97..7586e4af76983e785b2c6b3f03870a8bcd6df4a3 100644 --- a/docs/en/12-taos-sql/27-index.md +++ b/docs/en/12-taos-sql/27-index.md @@ -1,6 +1,7 @@ --- -sidebar_label: Index -title: Using Indices +title: Indexing +sidebar_label: Indexing +description: This document describes the SQL statements related to indexing in TDengine. --- TDengine supports SMA and FULLTEXT indexing. diff --git a/docs/en/12-taos-sql/28-recovery.md b/docs/en/12-taos-sql/28-recovery.md index e869ffc45ff7420836d91a2536d32ce6b08a0d82..b4da25ea0c3b4f2758c981ee5bd03985ef3f2754 100644 --- a/docs/en/12-taos-sql/28-recovery.md +++ b/docs/en/12-taos-sql/28-recovery.md @@ -1,6 +1,7 @@ --- -sidebar_label: Error Recovery title: Error Recovery +sidebar_label: Error Recovery +description: This document describes the SQL statements related to error recovery in TDengine. --- In a complex environment, connections and query tasks may encounter errors or fail to return in a reasonable time. If this occurs, you can terminate the connection or task. diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index 78b6d5fc05b9b03e1e8b3af268bc357dfaa401bc..341791d6755aa66fddd98561823ee0e5ac74ba3f 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -1,7 +1,7 @@ --- -sidebar_label: Changes in TDengine 3.0 title: Changes in TDengine 3.0 -description: "This document explains how TDengine SQL has changed in version 3.0." +sidebar_label: Changes in TDengine 3.0 +description: This document describes how TDengine SQL has changed in version 3.0 compared with previous versions. --- ## Basic SQL Elements @@ -54,7 +54,6 @@ The following data types can be used in the schema for standard tables. | 27 | GRANT | Added | Grants permissions to a user. | 28 | KILL TRANSACTION | Added | Terminates an mnode transaction. | 29 | KILL STREAM | Deprecated | Terminated a continuous query. The continuous query feature has been replaced with the stream processing feature. -| 30 | MERGE VGROUP | Added | Merges vgroups. | 31 | REVOKE | Added | Revokes permissions from a user. | 32 | SELECT | Modified |
  • SELECT does not use the implicit results column. Output columns must be specified in the SELECT clause.
  • DISTINCT support is enhanced. In previous versions, DISTINCT only worked on the tag column and could not be used with JOIN or GROUP BY.
  • JOIN support is enhanced. The following are now supported after JOIN: a WHERE clause with OR, operations on multiple tables, and GROUP BY on multiple tables.
  • Subqueries after FROM are enhanced. Levels of nesting are no longer restricted. Subqueries can be used with UNION ALL. Other syntax restrictions are eliminated.
  • All scalar functions can be used after WHERE.
  • GROUP BY is enhanced. You can group by any scalar expression or combination thereof.
  • SESSION can be used on supertables. When PARTITION BY is not used, data in supertables is merged into a single timeline.
  • STATE_WINDOW can be used on supertables. When PARTITION BY is not used, data in supertables is merged into a single timeline.
  • ORDER BY is enhanced. It is no longer required to use ORDER BY and GROUP BY together. There is no longer a restriction on the number of order expressions. NULLS FIRST and NULLS LAST syntax has been added. Any expression that conforms to the ORDER BY semantics can be used.
  • Added PARTITION BY syntax. PARTITION BY replaces GROUP BY tags.
| 33 | SHOW ACCOUNTS | Deprecated | This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." @@ -76,8 +75,9 @@ The following data types can be used in the schema for standard tables. | 49 | SHOW TRANSACTIONS | Added | Shows all running transactions in the system. | 50 | SHOW DNODE VARIABLES | Added | Shows the configuration of the specified dnode. | 51 | SHOW VNODES | Not supported | Shows information about vnodes in the system. Not supported. -| 52 | SPLIT VGROUP | Added | Splits a vgroup into two vgroups. -| 53 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 52 | TRIM DATABASE | Added | Deletes data that has expired and orders the remaining data in accordance with the storage configuration. +| 53 | REDISTRIBUTE VGROUP | Added | Adjust the distribution of VNODES in VGROUP. +| 54 | BALANCE VGROUP | Added | Auto adjust the distribution of VNODES in VGROUP. ## SQL Functions diff --git a/docs/en/12-taos-sql/index.md b/docs/en/12-taos-sql/index.md index a5ffc9dc8dce158eccc0fa0519f09ba346710c31..276f84f21bc8ab92d13f1975b298dbdcf25db249 100644 --- a/docs/en/12-taos-sql/index.md +++ b/docs/en/12-taos-sql/index.md @@ -1,6 +1,6 @@ --- title: TDengine SQL -description: 'The syntax supported by TDengine SQL ' +description: This document describes the syntax and functions supported by TDengine SQL. --- This section explains the syntax of SQL to perform operations on databases, tables and STables, insert data, select data and use functions. We also provide some tips that can be used in TDengine SQL. If you have previous experience with SQL this section will be fairly easy to understand. If you do not have previous experience with SQL, you'll come to appreciate the simplicity and power of SQL. TDengine SQL has been enhanced in version 3.0, and the query engine has been rearchitected. For information about how TDengine SQL has changed, see [Changes in TDengine 3.0](../taos-sql/changes). diff --git a/docs/en/13-operation/01-pkg-install.md b/docs/en/13-operation/01-pkg-install.md index d7713b943f5fe8fbd5e685b8ba03ff8cc8ed4e53..6e6c4aaebf7da8ed633cbb89233f9028f56d8302 100644 --- a/docs/en/13-operation/01-pkg-install.md +++ b/docs/en/13-operation/01-pkg-install.md @@ -1,6 +1,6 @@ --- title: Install and Uninstall -description: Install, Uninstall, Start, Stop and Upgrade +description: This document describes how to install, upgrade, and uninstall TDengine. --- import Tabs from "@theme/Tabs"; diff --git a/docs/en/13-operation/02-planning.mdx b/docs/en/13-operation/02-planning.mdx index 2dffa7bb8747e21e4754740208eafed65d341217..37ef6aae26ce55046ead09bcfbe37a14213bad46 100644 --- a/docs/en/13-operation/02-planning.mdx +++ b/docs/en/13-operation/02-planning.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: Resource Planning title: Resource Planning +sidebar_label: Resource Planning +description: This document describes how to plan compute and storage resources for your TDengine cluster. --- It is important to plan computing and storage resources if using TDengine to build an IoT, time-series or Big Data platform. How to plan the CPU, memory and disk resources required, will be described in this chapter. diff --git a/docs/en/13-operation/03-tolerance.md b/docs/en/13-operation/03-tolerance.md index 21a5a902822d7b85f555114a112686d4e35c64aa..4f33748e68a0b51be619fe7aa8cc3b2152c71bf2 100644 --- a/docs/en/13-operation/03-tolerance.md +++ b/docs/en/13-operation/03-tolerance.md @@ -1,5 +1,6 @@ --- title: Fault Tolerance and Disaster Recovery +description: This document describes how TDengine provides fault tolerance and disaster recovery. --- ## Fault Tolerance diff --git a/docs/en/13-operation/07-import.md b/docs/en/13-operation/07-import.md index 8362cec1ab3072866018678b42a679d0c19b49de..e95824e927f4c65b427d0819d7bc968c52f5fbe5 100644 --- a/docs/en/13-operation/07-import.md +++ b/docs/en/13-operation/07-import.md @@ -1,5 +1,6 @@ --- title: Data Import +description: This document describes how to import data into TDengine. --- There are multiple ways of importing data provided by TDengine: import with script, import from data file, import using `taosdump`. diff --git a/docs/en/13-operation/08-export.md b/docs/en/13-operation/08-export.md index 5780de42faeaedbc1c985ad2aa2f52fe56c76971..bffda36e23741a7ff52394772c918fa7040ea047 100644 --- a/docs/en/13-operation/08-export.md +++ b/docs/en/13-operation/08-export.md @@ -1,5 +1,6 @@ --- title: Data Export +description: This document describes how to export data from TDengine. --- There are two ways of exporting data from a TDengine cluster: diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 74a5564a2a7b82009e6b2406290a5c15afbf3c1d..346b874059a11c5608027b4ea6e2550765d64186 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -1,5 +1,6 @@ --- title: TDengine Monitoring +description: This document describes how to monitor your TDengine cluster. --- After TDengine is started, it automatically writes monitoring data including CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, into a designated database at a predefined interval through taosKeeper. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console. diff --git a/docs/en/13-operation/17-diagnose.md b/docs/en/13-operation/17-diagnose.md index fa202a23ea5b5d58eb0fc107045a9c27c7fa99ad..9d42b3ebbcfe0b87fa23bccaee12769d465fc24e 100644 --- a/docs/en/13-operation/17-diagnose.md +++ b/docs/en/13-operation/17-diagnose.md @@ -1,5 +1,6 @@ --- title: Problem Diagnostics +description: This document describes how to diagnose issues with your TDengine cluster. --- ## Network Connection Diagnostics diff --git a/docs/en/13-operation/index.md b/docs/en/13-operation/index.md index c64749c40e26f091e4a25e0238827ebceff4b069..8b386dc19a1113a84016bf3d03dd847216eed930 100644 --- a/docs/en/13-operation/index.md +++ b/docs/en/13-operation/index.md @@ -1,5 +1,6 @@ --- title: Administration +description: This document describes how to perform management operations on your TDengine cluster from an administrator's perspective. --- This chapter is mainly written for system administrators. It covers download, install/uninstall, data import/export, system monitoring, user management, connection management, capacity planning and system optimization. diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 09e40b956f9a3b713335bec768fd1d5474f4bea1..b138d69bfc277401723e5b75b874b42914963f48 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -1,5 +1,6 @@ --- title: REST API +description: This document describes the TDengine REST API. --- To support the development of various types of applications and platforms, TDengine provides an API that conforms to REST principles; namely REST API. To minimize the learning cost, unlike REST APIs for other database engines, TDengine allows insertion of SQL commands in the BODY of an HTTP POST request, to operate the database. diff --git a/docs/en/14-reference/03-connector/03-cpp.mdx b/docs/en/14-reference/03-connector/03-cpp.mdx index 906d56ab158e20db4bafa9eb8375434568555184..3bd7b7f4c6ad0c2ca3cf5c05fa72d1b27f714653 100644 --- a/docs/en/14-reference/03-connector/03-cpp.mdx +++ b/docs/en/14-reference/03-connector/03-cpp.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: C/C++ title: C/C++ Connector +sidebar_label: C/C++ +description: This document describes the TDengine C/C++ connector. --- C/C++ developers can use TDengine's client driver and the C/C++ connector, to develop their applications to connect to TDengine clusters for data writing, querying, and other functions. To use the C/C++ connector you must include the TDengine header file _taos.h_, which lists the function prototypes of the provided APIs. The application also needs to link to the corresponding dynamic libraries on the platform where it is located. diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 7b3440ebac2f630a2f6f39c3b270a2196d1d7739..36992da636a6773ede98ebca2ed6b981187afa93 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -1,8 +1,8 @@ --- -toc_max_heading_level: 4 -sidebar_label: Java title: TDengine Java Connector -description: The TDengine Java Connector is implemented on the standard JDBC API and provides native and REST connectors. +sidebar_label: Java +description: This document describes the TDengine Java Connector. +toc_max_heading_level: 4 --- import Tabs from '@theme/Tabs'; @@ -696,6 +696,9 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: Specifies whether to commit automatically. - group.id: consumer: Specifies the group that the consumer is in. - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set. +- td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni` +- httpConnectTimeout:WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. +- messageWaitTimeout:socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. - For more information, see [Consumer Parameters](../../../develop/tmq). #### Subscribe to consume data @@ -724,6 +727,11 @@ For more information, see [Data Subscription](../../../develop/tmq). ### Usage examples + + + +In addition to the native connection, the Java Connector also supports subscribing via websocket. + ```java public abstract class ConsumerLoop { private final TaosConsumer consumer; @@ -795,6 +803,87 @@ public abstract class ConsumerLoop { } ``` + + + +```java +public abstract class ConsumerLoop { + private final TaosConsumer consumer; + private final List topics; + private final AtomicBoolean shutdown; + private final CountDownLatch shutdownLatch; + + public ConsumerLoop() throws SQLException { + Properties config = new Properties(); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.type", "ws"); + config.setProperty("msg.with.table.name", "true"); + config.setProperty("enable.auto.commit", "true"); + config.setProperty("group.id", "group2"); + config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + + this.consumer = new TaosConsumer<>(config); + this.topics = Collections.singletonList("topic_speed"); + this.shutdown = new AtomicBoolean(false); + this.shutdownLatch = new CountDownLatch(1); + } + + public abstract void process(ResultBean result); + + public void pollData() throws SQLException { + try { + consumer.subscribe(topics); + + while (!shutdown.get()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } + } + consumer.unsubscribe(); + } finally { + consumer.close(); + shutdownLatch.countDown(); + } + } + + public void shutdown() throws InterruptedException { + shutdown.set(true); + shutdownLatch.await(); + } + + public static class ResultDeserializer extends ReferenceDeserializer { + + } + + public static class ResultBean { + private Timestamp ts; + private int speed; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public int getSpeed() { + return speed; + } + + public void setSpeed(int speed) { + this.speed = speed; + } + } +} +``` + + + + +> **Note**: The value of value.deserializer should be adjusted based on the package path of the test environment. + ### Use with connection pool #### HikariCP @@ -878,8 +967,10 @@ The source code of the sample application is under `TDengine/examples/JDBC`: | taos-jdbcdriver version | major changes | | :---------------------: | :--------------------------------------------: | -| 3.0.1 - 3.0.2 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use 3.0.2 in the JDK 8 environment | +| 3.1.0 | JDBC REST connection supports subscription over WebSocket | +| 3.0.1 - 3.0.4 | fix the resultSet data is parsed incorrectly sometimes. 3.0.1 is compiled on JDK 11, you are advised to use other version in the JDK 8 environment | | 3.0.0 | Support for TDengine 3.0 | +| 2.0.42 | fix wasNull interface return value in WebSocket connection | | 2.0.41 | fix decode method of username and password in REST connection | | 2.0.39 - 2.0.40 | Add REST connection/request timeout parameters | | 2.0.38 | JDBC REST connections add bulk pull function | diff --git a/docs/en/14-reference/03-connector/05-go.mdx b/docs/en/14-reference/03-connector/05-go.mdx index df5b129cea552144d5833190d46e8a78f2fd2fa5..da2f54708f550bc4ba9f35da81ad59a575bda169 100644 --- a/docs/en/14-reference/03-connector/05-go.mdx +++ b/docs/en/14-reference/03-connector/05-go.mdx @@ -1,7 +1,8 @@ --- -toc_max_heading_level: 4 -sidebar_label: Go title: TDengine Go Connector +sidebar_label: Go +description: This document describes the TDengine Go connector. +toc_max_heading_level: 4 --- import Tabs from '@theme/Tabs'; @@ -355,26 +356,29 @@ The `af` package encapsulates TDengine advanced functions such as connection man #### Subscribe -* `func NewConsumer(conf *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` Creates consumer group. -* `func (c *Consumer) Subscribe(topics []string) error` +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose + +Subscribes a topic. + +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose Subscribes to topics. -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` Polling information. -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose Commit information. -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` - -Free information. - * `func (c *Consumer) Unsubscribe() error` Unsubscribe. @@ -441,25 +445,36 @@ Close consumer. ### Subscribe via WebSocket -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` - Creates consumer group. +Creates consumer group. -* `func (c *Consumer) Subscribe(topic []string) error` +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose - Subscribes to topics. +Subscribes a topic. -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +Note: `rebalanceCb` is reserved for compatibility purpose - Polling information. +Subscribes to topics. -* `func (c *Consumer) Commit(messageID uint64) error` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - Commit information. +Polling information. + +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +Note: `tmq.TopicPartition` is reserved for compatibility purpose + +Commit information. + +* `func (c *Consumer) Unsubscribe() error` + +Unsubscribe. * `func (c *Consumer) Close() error` - Close consumer. +Close consumer. For a complete example see [GitHub sample file](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 4e2a7848dc87ac0522b6d5aa5855b5a0998dc789..6e848596107f3dd8a0e99b4bf8fe1dc9530021a4 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -1,7 +1,8 @@ --- -toc_max_heading_level: 4 -sidebar_label: Rust title: TDengine Rust Connector +sidebar_label: Rust +description: This document describes the TDengine Rust connector. +toc_max_heading_level: 4 --- import Tabs from '@theme/Tabs'; diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index 25e6b2188a64928e35b8e8c45988a426802eb9f3..146da268a8b7b4ebf799f3441efcd2d454afd65d 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -1,7 +1,7 @@ --- -sidebar_label: Python title: TDengine Python Connector -description: "taospy is the official Python connector for TDengine. taospy provides a rich API that makes it easy for Python applications to use TDengine. tasopy wraps both the native and REST interfaces of TDengine, corresponding to the two submodules of tasopy: taos and taosrest. In addition to wrapping the native and REST interfaces, taospy also provides a programming interface that conforms to the Python Data Access Specification (PEP 249), making it easy to integrate taospy with many third-party tools, such as SQLAlchemy and pandas." +sidebar_label: Python +description: This document describes taospy, the TDengine Python connector. --- import Tabs from "@theme/Tabs"; @@ -32,7 +32,7 @@ We recommend using the latest version of `taospy`, regardless of the version of ### Preparation -1. Install Python. Python >= 3.7 is recommended. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it. +1. Install Python. The recent taospy package requires Python 3.6+. The earlier versions of taospy require Python 3.7+. The taos-ws-py package requires Python 3.7+. If Python is not available on your system, refer to the [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) to install it. 2. Install [pip](https://pypi.org/project/pip/). In most cases, the Python installer comes with the pip utility. If not, please refer to [pip documentation](https://pip.pypa.io/en/stable/installation/) to install it. If you use a native connection, you will also need to [Install Client Driver](/reference/connector#Install-Client-Driver). The client install package includes the TDengine client dynamic link library (`libtaos.so` or `taos.dll`) and the TDengine CLI. @@ -78,6 +78,22 @@ pip3 install git+https://github.com/taosdata/taos-connector-python.git
+#### Install `taos-ws-py` (Optional) + +The taos-ws-py package provides the way to access TDengine via WebSocket. + +##### Install taos-ws-py with taospy + +```bash +pip3 install taospy[ws] +``` + +##### Install taos-ws-py only + +```bash +pip3 install taos-ws-py +``` + ### Verify diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx index a36cf0efc9432425ad16c4d8112cc813a8c528b9..b7c2e6b76537dc8b2a039684d2d712d91337cecd 100644 --- a/docs/en/14-reference/03-connector/08-node.mdx +++ b/docs/en/14-reference/03-connector/08-node.mdx @@ -1,7 +1,8 @@ --- -toc_max_heading_level: 4 -sidebar_label: Node.js title: TDengine Node.js Connector +sidebar_label: Node.js +description: This document describes the TDengine Node.js connector. +toc_max_heading_level: 4 --- import Tabs from "@theme/Tabs"; @@ -31,7 +32,9 @@ Please refer to [version support list](/reference/connector#version-support) ## Supported features -### Native connectors + + + 1. Connection Management 2. General Query @@ -40,12 +43,16 @@ Please refer to [version support list](/reference/connector#version-support) 5. Subscription 6. Schemaless -### REST Connector + + 1. Connection Management 2. General Query 3. Continuous Query + + + ## Installation Steps ### Pre-installation preparation @@ -59,9 +66,19 @@ Please refer to [version support list](/reference/connector#version-support) - `python` (recommended for `v2.7` , `v3.x.x` currently not supported) - `@tdengine/client` 3.0.0 supports Node.js LTS v10.9.0 or later and Node.js LTS v12.8.0 or later. Older versions may be incompatible. - `make` -- C compiler, [GCC](https://gcc.gnu.org) v4.8.5 or higher +- C compiler, [GCC](https://gcc.gnu.org) v4.8.5 or later. + + + +- `python` (recommended for `v2.7` , `v3.x.x` currently not supported) +- `@tdengine/client` 3.0.0 currently supports Node.js from v12.22.12, but only later versions of v12. Other versions may be incompatible. +- `make` +- C compiler, [GCC](https://gcc.gnu.org) v4.8.5 or later. + + + - Installation method 1 @@ -104,6 +121,9 @@ npm install @tdengine/rest ### Verify + + + After installing the TDengine client, use the `nodejsChecker.js` program to verify that the current environment supports Node.js access to TDengine. Verification in details: @@ -120,6 +140,28 @@ node nodejsChecker.js host=localhost - After executing the above steps, the command-line will output the result of `nodejsChecker.js` connecting to the TDengine instance and performing a simple insert and query. + + + +After installing the TDengine client, use the `restChecker.js` program to verify that the current environment supports Node.js access to TDengine. + +Verification in details: + +- Create an installation test folder such as `~/tdengine-test`. Download the [restChecker.js source code](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/node/restexample/restChecker.js) to your local. + +- Execute the following command from the command-line. + +```bash +npm init -y +npm install @tdengine/rest +node restChecker.js +``` + +- After executing the above steps, the command-line will output the result of `restChecker.js` connecting to the TDengine instance and performing a simple insert and query. + + + + ## Establishing a connection Please choose to use one of the connectors. @@ -171,24 +213,69 @@ let cursor = conn.cursor(); #### SQL Write + + + + + + +```js +{{#include docs/examples/node/restexample/insert_example.js}} +``` + + + + #### InfluxDB line protocol write + + + + + + #### OpenTSDB Telnet line protocol write + + + + + + #### OpenTSDB JSON line protocol write + + + + + + ### Querying data + + + + + + + +```js +{{#include docs/examples/node/restexample/query_example.js}} +``` + + + + ## More sample programs @@ -249,4 +336,4 @@ let cursor = conn.cursor(); ## API Reference -[API Reference](https://docs.taosdata.com/api/td2.0-connector/) \ No newline at end of file +[API Reference](https://docs.taosdata.com/api/td2.0-connector/) diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index 85514f58ac1a19c7ae1a725e9b055f10280ebbb6..e984967fb7048bf4a89d5528be7f6c977820d255 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -1,7 +1,8 @@ --- -toc_max_heading_level: 4 -sidebar_label: C# title: C# Connector +sidebar_label: C# +description: This document describes the TDengine C# connector. +toc_max_heading_level: 4 --- import Tabs from '@theme/Tabs'; @@ -17,7 +18,7 @@ import CSAsyncQuery from "../../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` is a C# language connector provided by TDengine that allows C# developers to develop C# applications that access TDengine cluster data. -The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc.The `TDengine.Connector` also supports WebSocket and developers can build connection through DSN, which supports data writing, querying, and parameter binding, etc. +The `TDengine.Connector` connector supports connect to TDengine instances via the TDengine client driver (taosc), providing data writing, querying, subscription, schemaless writing, bind interface, etc.The `TDengine.Connector` also supports WebSocket from v3.0.1 and developers can build connection through DSN, which supports data writing, querying, and parameter binding, etc. This article describes how to install `TDengine.Connector` in a Linux or Windows environment and connect to TDengine clusters via `TDengine.Connector` to perform basic operations such as data writing and querying. @@ -66,31 +67,43 @@ Please refer to [version support list](/reference/connector#version-support) * [Nuget Client](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) (optional installation) * Install TDengine client driver, please refer to [Install client driver](/reference/connector/#install-client-driver) for details -### Install via dotnet CLI +### Install `TDengine.Connector` - + -You can reference the `TDengine.Connector` published in Nuget to the current project via the `dotnet` command under the path of the existing .NET project. +You can reference the `TDengine.Connector` published in Nuget to the current project via the `dotnet` CLI under the path of the existing .NET project. ``` bash dotnet add package TDengine.Connector ``` - - +You may also modify the current.NET project file. You can include the following 'ItemGroup' in your project file (.csproj). -You can [download the source code](https://github.com/taosdata/taos-connector-dotnet/tree/3.0) and directly reference the latest version of the TDengine.Connector library. - -```bash -git clone -b 3.0 https://github.com/taosdata/taos-connector-dotnet.git -cd taos-connector-dotnet -cp -r src/ myProject +``` XML + + + +``` -cd myProject -dotnet add exmaple.csproj reference src/TDengine.csproj + + + +In this scenario, modifying your project file is required in order to copy the WebSocket dependency dynamic library from the nuget package into your project. +```XML + + + + + + + + + ``` +Notice: `TDengine.Connector` only version>= 3.0.2 includes the dynamic library for WebSocket. + @@ -252,19 +265,20 @@ ws://localhost:6041/test |Sample program |Sample program description | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | Parameter binding with TDengine Connector | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | Schemaless writes with TDengine Connector | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | -| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | -| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | Table creation, data insertion, and query examples with TDengine.Connector | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | Writing and querying JSON tag data with TDengine Connector | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | Parameter binding with TDengine Connector | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | Schemaless writes with TDengine Connector | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | Asynchronous queries with TDengine Connector | +| [Subscription](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | Subscription example with TDengine Connector | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | WebSocket basic data in and out with TDengine connector | +| [WebSocket Parameter Binding](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | WebSocket parameter binding example | ## Important update records | TDengine.Connector | Description | |--------------------|--------------------------------| +| 3.0.2 | Support .NET Framework 4.5 and above. Support .Net standard 2.0. Nuget package includes dynamic library for WebSocket.| | 3.0.1 | Support WebSocket and Cloud,With function query, insert, and parameter binding| | 3.0.0 | Supports TDengine 3.0.0.0. TDengine 2.x is not supported. Added `TDengine.Impl.GetData()` interface to deserialize query results. | | 1.0.7 | Fixed TDengine.Query() memory leak. | diff --git a/docs/en/14-reference/03-connector/10-php.mdx b/docs/en/14-reference/03-connector/10-php.mdx index 87f8616f9ebac598285406eb404b7c2544950ee3..fd00d11239315bbab6e59d838d472d3496ff8342 100644 --- a/docs/en/14-reference/03-connector/10-php.mdx +++ b/docs/en/14-reference/03-connector/10-php.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: PHP title: PHP Connector +sidebar_label: PHP +description: This document describes the TDengine PHP connector. --- `php-tdengine` is the TDengine PHP connector provided by TDengine community. In particular, it supports Swoole coroutine. diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index ba8dbb85d4982c5d6c89f5dbe6157bd88a8c00a4..2df73b8592c496ca1e3e715c86a9fbb83420f425 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -1,5 +1,6 @@ --- title: Connector +description: This document describes the connectors that TDengine provides to interface with various programming languages. --- TDengine provides a rich set of APIs (application development interface). To facilitate users to develop their applications quickly, TDengine supports connectors for multiple programming languages, including official connectors for C/C++, Java, Python, Go, Node.js, C#, and Rust. These connectors support connecting to TDengine clusters using both native interfaces (taosc) and REST interfaces (not supported in a few languages yet). Community developers have also contributed several unofficial connectors, such as the ADO.NET connector, the Lua connector, and the PHP connector. @@ -59,11 +60,11 @@ The different database framework specifications for various programming language | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | | **Connection Management** | Support | Support | Support | Support | Support | Support | | **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Not supported | Not supported | support | Support | Not supported | Support | -| **Subscription (TMQ) ** | Not supported | Not supported | support | Not supported | Not supported | Support | -| **Schemaless** | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | -| **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | support | Support | Support | -| **DataFrame** | Not supported | Support | Not supported | Not supported | Not supported | Not supported | +| **Parameter Binding** | Not Supported | Not Supported | Support | Support | Not Supported | Support | +| **Subscription (TMQ) ** | Not Supported | Support | Support | Not Supported | Not Supported | Support | +| **Schemaless** | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | +| **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | Support | Support | Support | +| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | :::warning diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index ad00584360acc18dca46998948fd96790965e5b1..c1ec97b647c228b7302391f768033792e236c79b 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -1,7 +1,7 @@ --- -title: "taosAdapter" -description: "taosAdapter is a TDengine companion tool that acts as a bridge and adapter between TDengine clusters and applications. It provides an easy-to-use and efficient way to ingest data directly from data collection agent software such as Telegraf, StatsD, collectd, etc. It also provides an InfluxDB/OpenTSDB compatible data ingestion interface, allowing InfluxDB/OpenTSDB applications to be seamlessly ported to TDengine." -sidebar_label: "taosAdapter" +title: taosAdapter +sidebar_label: taosAdapter +description: This document describes how to use taosAdapter, a TDengine companion tool that acts as a bridge and adapter between TDengine clusters and applications. --- import Prometheus from "./_prometheus.mdx" @@ -21,6 +21,7 @@ taosAdapter provides the following features. - Seamless connection to collectd - Seamless connection to StatsD - Supports Prometheus remote_read and remote_write +- Get table's VGroup ID ## taosAdapter architecture diagram @@ -59,6 +60,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +102,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +113,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +135,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -174,6 +179,7 @@ See [example/config/taosadapter.toml](https://github.com/taosdata/taosadapter/bl node_export is an exporter for machine metrics. Please visit [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) for more information. - Support for Prometheus remote_read and remote_write remote_read and remote_write are interfaces for Prometheus data read and write from/to other data storage solution. Please visit [https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) for more information. +- Get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). ## Interfaces @@ -195,6 +201,7 @@ Support InfluxDB query parameters as follows. - `precision` The time precision used by TDengine - `u` TDengine user name - `p` TDengine password +- `ttl` The time to live of automatically created sub-table. This value cannot be updated. TDengine will use the ttl value of the first data of sub-table to create sub-table. For more information, please refer [Create Table](/taos-sql/table/#create-table) Note: InfluxDB token authorization is not supported at present. Only Basic authorization and query parameter validation are supported. Example: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -236,6 +243,10 @@ node_export is an exporter of hardware and OS metrics exposed by the \*NIX kerne +### Get table's VGroup ID + +You can call `http://:6041/rest/vgid?db=&table=` to get table's VGroup ID. For more information about VGroup, please refer to [primary-logic-unit](/tdinternal/arch/#primary-logic-unit). + ## Memory usage optimization methods taosAdapter will monitor its memory usage during operation and adjust it with two thresholds. Valid values are integers between 1 to 100, and represent a percentage of the system's physical memory. diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 6e08671e344100aa3dcfeba5a10c0d0e304a1a91..9828d71ece98ca6de2f079f5599f2eed7fcea891 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -1,8 +1,8 @@ --- title: taosBenchmark sidebar_label: taosBenchmark +description: This document describes how to use taosBenchmark, a tool for testing the performance of TDengine. toc_max_heading_level: 4 -description: "taosBenchmark (once called taosdemo ) is a tool for testing the performance of TDengine." --- # Introduction @@ -92,7 +92,7 @@ taosBenchmark -f -## Command-line argument in detailed +## Command-line argument in detail - **-f/--file ** : specify the configuration file to use. This file includes All parameters. Users should not use this parameter with other parameters on the command-line. There is no default value. @@ -198,19 +198,25 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-R/--disorder-range ** : Specify the timestamp range for the disordered data. It leads the resulting disorder timestamp as the ordered timestamp minus a random value in this range. Valid only if the percentage of disordered data specified by `-O/--disorder` is greater than 0. -- **-F/--prepare_rand ** : +- **-F/--prepared_rand ** : Specify the number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. - **-a/--replica ** : Specify the number of replicas when creating the database. The default value is 1. +- **-k/--keep-trying ** : + Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- **-z/--trying-interval ** : + Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + - **-V/--version** : Show version information only. Users should not use it with other parameters. - **-? /--help** : Show help information and exit. Users should not use it with other parameters. -## Configuration file parameters in detailed +## Configuration file parameters in detail ### General configuration parameters @@ -231,6 +237,10 @@ The parameters listed in this section apply to all function modes. `filetype` must be set to `insert` in the insertion scenario. See [General Configuration Parameters](#General Configuration Parameters) +- ** keep_trying ** : Keep trying if failed to insert, default is no. Available with v3.0.9+. + +- ** trying_interval ** : Specify interval between keep trying insert. Valid value is a postive number. Only valid when keep trying be enabled. Available with v3.0.9+. + #### Database related configuration parameters The parameters related to database creation are configured in `dbinfo` in the json configuration file, as follows. The other parameters correspond to the database parameters specified when `create database` in [../../taos-sql/database]. @@ -370,7 +380,7 @@ The configuration parameters for specifying super table tag columns and data col - **num_of_records_per_req** : Writing the number of rows of records per request to TDengine, the default value is 30000. When it is set too large, the TDengine client driver will return the corresponding error message, so you need to lower the setting of this parameter to meet the writing requirements. -- **prepare_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. +- **prepared_rand**: The number of unique values in the generated random data. A value of 1 means that all data are equal. The default value is 10000. ### Query scenario configuration parameters diff --git a/docs/en/14-reference/06-taosdump.md b/docs/en/14-reference/06-taosdump.md index e73441a96b087062b2e3912ed73010fc3e761bb9..dcfc068a3d64eb9f755f498b2b80d0a2b228842f 100644 --- a/docs/en/14-reference/06-taosdump.md +++ b/docs/en/14-reference/06-taosdump.md @@ -1,6 +1,6 @@ --- title: taosdump -description: "taosdump is a tool that supports backing up data from a running TDengine cluster and restoring the backed up data to the same, or another running TDengine cluster." +description: This document describes how to use taosdump, a tool for backing up and restoring the data in a TDengine cluster. --- ## Introduction @@ -19,7 +19,7 @@ Users should not use taosdump to back up raw data, environment settings, hardwar There are two ways to install taosdump: -- Install the taosTools official installer. Please find taosTools from [All download links](https://www.tdengine.com/all-downloads) page and download and install it. +- Install the taosTools official installer. Please find taosTools from [Release History](https://docs.taosdata.com/releases/tools/) page and download and install it. - Compile taos-tools separately and install it. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details. diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp index 942130d4fabf7944c7add10acb3bb42ca7f51e0f..a90477880ff3a2cb2068540b55b69e4bbdbd8d4c 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp differ diff --git a/docs/en/14-reference/07-tdinsight/index.md b/docs/en/14-reference/07-tdinsight/index.md index d03c16a8bc25a39fb751ded4b5c12df4b69a8e4e..652d5ded0b3fed28961e92d4894fa090be393873 100644 --- a/docs/en/14-reference/07-tdinsight/index.md +++ b/docs/en/14-reference/07-tdinsight/index.md @@ -1,6 +1,7 @@ --- title: TDinsight - Grafana-based Zero-Dependency Monitoring Solution for TDengine sidebar_label: TDinsight +description: This document describes TDinsight, a monitoring solution for TDengine. --- TDinsight is a solution for monitoring TDengine using the builtin native monitoring database and [Grafana]. @@ -325,11 +326,12 @@ Currently, only the number of logins per minute is reported. Support monitoring taosAdapter request statistics and status details. Includes. -1. **http_request_inflight**: number of real-time requests. -2. **http_request_total**: number of total requests. -3. **http_request_fail**: number of failed requets. -4. **CPU Used**: CPU usage of taosAdapter. -5. **Memory Used**: Memory usage of taosAdapter. +1. **Http Request Total**: number of total requests. +2. **Http Request Fail**: number of failed requests. +3. **CPU Used**: CPU usage of taosAdapter. +4. **Memory Used**: Memory usage of taosAdapter. +5. **Http Request Inflight**: number of real-time requests. +6. **Http Status Code**: taosAdapter http status code. ## Upgrade diff --git a/docs/en/14-reference/08-taos-shell.md b/docs/en/14-reference/08-taos-shell.md index 68e2f087658305ec408c50e27f4a87ef43b2c870..7833ac861fea5fb2dfd66ccc9ec2db8f0ddeff3f 100644 --- a/docs/en/14-reference/08-taos-shell.md +++ b/docs/en/14-reference/08-taos-shell.md @@ -1,7 +1,7 @@ --- title: TDengine Command Line Interface (CLI) sidebar_label: Command Line Interface -description: Instructions and tips for using the TDengine CLI +description: This document describes how to use the TDengine CLI. --- The TDengine command-line interface (hereafter referred to as `TDengine CLI`) is the simplest way for users to manipulate and interact with TDengine instances. diff --git a/docs/en/14-reference/09-support-platform/index.md b/docs/en/14-reference/09-support-platform/index.md index 061294f0160f5f07c7c032b4a7475f6b12c2efca..7dfa8ac93a9e91eb9238f7b56033a592c241079f 100644 --- a/docs/en/14-reference/09-support-platform/index.md +++ b/docs/en/14-reference/09-support-platform/index.md @@ -1,6 +1,6 @@ --- title: List of supported platforms -description: "List of platforms supported by TDengine server, client, and connector" +description: This document describes the supported platforms for the TDengine server, client, and connectors. --- ## List of supported platforms for TDengine server diff --git a/docs/en/14-reference/11-docker/index.md b/docs/en/14-reference/11-docker/index.md index 89987c456f14d243a4b59af5b15ae754225bbaf9..5a48f2e4b1e58132c728dadfba2afbe39ae8bef3 100644 --- a/docs/en/14-reference/11-docker/index.md +++ b/docs/en/14-reference/11-docker/index.md @@ -1,6 +1,6 @@ --- title: Deploying TDengine with Docker -description: "This chapter focuses on starting the TDengine service in a container and accessing it." +description: This chapter describes how to start and access TDengine in a Docker container. --- This chapter describes how to start the TDengine service in a container and access it. Users can control the behavior of the service in the container by using environment variables on the docker run command-line or in the docker-compose file. @@ -273,49 +273,48 @@ password: taosdata ## Start the TDengine cluster with docker-compose -1. The following docker-compose file starts a TDengine cluster with two replicas, two management nodes, two data nodes, and one arbitrator. - - ```docker - version: "3" - services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - td-1: - image: tdengine/tdengine:$VERSION - environment: - TAOS_FQDN: "td-1" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ - td-2: - image: tdengine/tdengine:$VERSION - environment: - TAOS_FQDN: "td-2" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ - volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - ``` +1. The following docker-compose file starts a TDengine cluster with three nodes. + +```yml +version: "3" +services: + td-1: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + td-3: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-3" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td3:/var/lib/taos/ + - taoslog-td3:/var/log/taos/ +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: + taosdata-td3: + taoslog-td3: +``` :::note - The `VERSION` environment variable is used to set the tdengine image tag - `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time -- `TAOS_REPLICA` is used to set the default number of database replicas. Its value range is [1,3] - We recommend setting it with `TAOS_ARBITRATOR` to use arbitrator in a two-nodes environment. ::: 2. Start the cluster @@ -345,17 +344,18 @@ password: taosdata 4. Show dnodes via TDengine CLI - ```shell - $ docker-compose exec td-1 taos -s "show dnodes" - - taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | - ====================================================================================================================================== - 1 | td-1:6030 | 1 | 8 | ready | any | 2022-01-18 02:47:42.871 | | - 2 | td-2:6030 | 0 | 8 | ready | any | 2022-01-18 02:47:43.518 | | - 0 | arbitrator:6042 | 0 | 0 | ready | arb | 2022-01-18 02:47:43.633 | - | - Query OK, 3 row(s) in set (0.000811s) - ``` +```shell +$ docker-compose exec td-1 taos -s "show dnodes" + +taos> show dnodes + id | endpoint | vnodes | support_vnodes | status | create_time | note | +====================================================================================================================================== + 1 | td-1:6030 | 0 | 32 | ready | 2022-08-19 07:57:29.971 | | + 2 | td-2:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.415 | | + 3 | td-3:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.417 | | +Query OK, 3 rows in database (0.021262s) + +``` ## taosAdapter @@ -373,83 +373,70 @@ password: taosdata Suppose you want to deploy multiple taosAdapters to improve throughput and provide high availability. In that case, the recommended configuration method uses a reverse proxy such as Nginx to offer a unified access entry. For specific configuration methods, please refer to the official documentation of Nginx. Here is an example: - ```docker - version: "3" - - networks: - inter: - api: - - services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - networks: - - inter - td-1: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-1" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ - td-2: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-2" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ - adapter: - image: tdengine/tdengine:$VERSION - command: taosadapter - networks: - - inter - environment: - TAOS_FIRST_EP: "td-1" - TAOS_SECOND_EP: "td-2" - deploy: - replicas: 4 - nginx: - image: nginx - depends_on: - - adapter - networks: - - inter - - api - ports: - - 6041:6041 - - 6044:6044/udp - command: [ - "sh", - "-c", - "while true; - do curl -s http://adapter:6041/-/ping >/dev/null && break; - done; - printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' - > /etc/nginx/conf.d/rest.conf; - printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' - >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; - nginx -g 'daemon off;'", - ] - volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: - ``` +```yml +version: "3" + +networks: + inter: + +services: + td-1: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-1" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td1:/var/lib/taos/ + - taoslog-td1:/var/log/taos/ + td-2: + image: tdengine/tdengine:$VERSION + networks: + - inter + environment: + TAOS_FQDN: "td-2" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td2:/var/lib/taos/ + - taoslog-td2:/var/log/taos/ + adapter: + image: tdengine/tdengine:$VERSION + entrypoint: "taosadapter" + networks: + - inter + environment: + TAOS_FIRST_EP: "td-1" + TAOS_SECOND_EP: "td-2" + deploy: + replicas: 4 + nginx: + image: nginx + depends_on: + - adapter + networks: + - inter + ports: + - 6041:6041 + - 6044:6044/udp + command: [ + "sh", + "-c", + "while true; + do curl -s http://adapter:6041/-/ping >/dev/null && break; + done; + printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' + > /etc/nginx/conf.d/rest.conf; + printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' + >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; + nginx -g 'daemon off;'", + ] +volumes: + taosdata-td1: + taoslog-td1: + taosdata-td2: + taoslog-td2: +``` ## Deploy with docker swarm diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 274c99509836702dc067ca7f3357acc96be4132c..7075dc771eccc83a7cabb6807f34667de8f5fcc7 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -1,6 +1,6 @@ --- title: Configuration Parameters -description: "Configuration parameters for client and server in TDengine" +description: This document describes the configuration parameters for the TDengine server and client. --- ## Configuration File on Server Side @@ -106,7 +106,7 @@ The parameters described in this document by the effect that they have on the sy | Applicable | Server only | | Meaning | The switch for monitoring inside server. The main object of monitoring is to collect information about load on physical nodes, including CPU usage, memory usage, disk usage, and network bandwidth. Monitoring information is sent over HTTP to the taosKeeper service specified by `monitorFqdn` and `monitorProt`. | Value Range | 0: monitoring disabled, 1: monitoring enabled | -| Default | 1 | +| Default | 0 | ### monitorFqdn @@ -142,6 +142,15 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Switch for allowing TDengine to collect and report service usage information | | Value Range | 0: Not allowed; 1: Allowed | | Default Value | 1 | +### crashReporting + +| Attribute | Description | +| -------- | -------------------------------------------- | +| Applicable | Server Only | +| Meaning |Switch for allowing TDengine to collect and report crash related information | +| Value Range | 0,1 0: Not allowed;1:allowed | +| Default Value | 1 | + ## Query Parameters @@ -153,11 +162,7 @@ The parameters described in this document by the effect that they have on the sy | Meaning | Execution policy for query statements | | Unit | None | | Default | 1 | -| Notes | 1: Run queries on vnodes and not on qnodes | - -2: Run subtasks without scan operators on qnodes and subtasks with scan operators on vnodes. - -3: Only run scan operators on vnodes; run all other operators on qnodes. +| Value Range | 1: Run queries on vnodes and not on qnodes; 2: Run subtasks without scan operators on qnodes and subtasks with scan operators on vnodes; 3: Only run scan operators on vnodes, and run all other operators on qnodes. | ### querySmaOptimize @@ -167,12 +172,17 @@ The parameters described in this document by the effect that they have on the sy | Meaning | SMA index optimization policy | | Unit | None | | Default Value | 0 | -| Notes | - -0: Disable SMA indexing and perform all queries on non-indexed data. +| Notes |0: Disable SMA indexing and perform all queries on non-indexed data; 1: Enable SMA indexing and perform queries from suitable statements on precomputation results.| -1: Enable SMA indexing and perform queries from suitable statements on precomputation results.| +### countAlwaysReturnValue +| Attribute | Description | +| -------- | -------------------------------- | +| Applicable | Server only | +| Meaning | count()/hyperloglog() return value or not if the input data is empty or NULL | +| Vlue Range | 0:Return empty line,1:Return 0 | +| Default | 1 | +| Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | ### maxNumOfDistinctRes @@ -306,6 +316,15 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | All data files are stored in this directory | | Default Value | /var/lib/taos | +| Note | The [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function needs to be used in conjunction with the [KEEP](https://docs.tdengine.com/taos-sql/database/#parameters) parameter | + +### tempDir + +| Attribute | Description | +| -------- | ------------------------------------------ | +| Applicable | Server only | +| Meaning | The directory where to put all the temporary files generated during system running | +| Default | /tmp | ### minimalTmpDirGB @@ -336,89 +355,6 @@ The charset that takes effect is UTF-8. | Value Range | 0-4096 | | Default Value | 2x the CPU cores | -## Time Parameters - -### statusInterval - -| Attribute | Description | -| -------- | --------------------------- | -| Applicable | Server Only | -| Meaning | the interval of dnode reporting status to mnode | -| Unit | second | -| Value Range | 1-10 | -| Default Value | 1 | - -### shellActivityTimer - -| Attribute | Description | -| -------- | --------------------------------- | -| Applicable | Server and Client | -| Meaning | The interval for TDengine CLI to send heartbeat to mnode | -| Unit | second | -| Value Range | 1-120 | -| Default Value | 3 | - -## Performance Optimization Parameters - -### numOfCommitThreads - -| Attribute | Description | -| -------- | ---------------------- | -| Applicable | Server Only | -| Meaning | Maximum of threads for committing to disk | -| Default Value | | - -## Compression Parameters - -### compressMsgSize - -| Attribute | Description | -| ------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for message size to compress the message. | Set the value to 64330 bytes for good message compression. | -| Unit | bytes | -| Value Range | 0: already compress; >0: compress when message exceeds it; -1: always uncompress | -| Default Value | -1 | - -### compressColData - -| Attribute | Description | -| -------- | --------------------------------------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | The threshold for size of column data to trigger compression for the query result | -| Unit | bytes | -| Value Range | 0: always compress; >0: only compress when the size of any column data exceeds the threshold; -1: always uncompress | -| Default Value | -1 | -| Default Value | -1 | -| Note | available from version 2.3.0.0 | | - -## Continuous Query Parameters | - -### minSlidingTime - -| Attribute | Description | -| ------------- | -------------------------------------------------------- | -| Applicable | Server Only | -| Meaning | Minimum sliding time of time window | -| Unit | millisecond or microsecond , depending on time precision | -| Value Range | 10-1000000 | -| Default Value | 10 | - -### minIntervalTime - -| Attribute | Description | -| ------------- | --------------------------- | -| Applicable | Server Only | -| Meaning | Minimum size of time window | -| Unit | millisecond | -| Value Range | 1-1000000 | -| Default Value | 10 | - -:::info -To prevent system resource from being exhausted by multiple concurrent streams, a random delay is applied on each stream automatically. `maxFirstStreamCompDelay` is the maximum delay time before a continuous query is started the first time. `streamCompDelayRatio` is the ratio for calculating delay time, with the size of the time window as base. `maxStreamCompDelay` is the maximum delay time. The actual delay time is a random time not bigger than `maxStreamCompDelay`. If a continuous query fails, `retryStreamComDelay` is the delay time before retrying it, also not bigger than `maxStreamCompDelay`. - -::: - ## Log Parameters ### logDir @@ -661,10 +597,22 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Attribute | Description | | -------- | ----------------------------- | | Applicable | Client only | -| Meaning | Whether schemaless columns are consistently ordered | +| Meaning | Whether schemaless columns are consistently ordered, depat, discarded since 3.0.3.0| | Value Range | 0: not consistent; 1: consistent. | | Default | 1 | +## Compress Parameters + +### compressMsgSize + +| Attribute | Description | +| -------- | ----------------------------- | +| Applicable | Both Client and Server side | +| Meaning | Whether RPC message is compressed | +| Value Range | -1: none message is compressed; 0: all messages are compressed; N (N>0): messages exceeding N bytes are compressed | +| Default | -1 | + + ## Other Parameters ### enableCoreFile @@ -686,172 +634,60 @@ To prevent system resource from being exhausted by multiple concurrent streams, | Value Range | 0: disable UDF; 1: enabled UDF | | Default Value | 1 | -## Parameter Comparison of TDengine 2.x and 3.0 -| # | **Parameter** | **In 2.x** | **In 3.0** | -| --- | :-----------------: | --------------- | --------------- | -| 1 | firstEp | Yes | Yes | -| 2 | secondEp | Yes | Yes | -| 3 | fqdn | Yes | Yes | -| 4 | serverPort | Yes | Yes | -| 5 | maxShellConns | Yes | Yes | -| 6 | monitor | Yes | Yes | -| 7 | monitorFqdn | No | Yes | -| 8 | monitorPort | No | Yes | -| 9 | monitorInterval | Yes | Yes | -| 10 | monitorMaxLogs | No | Yes | -| 11 | monitorComp | No | Yes | -| 12 | telemetryReporting | Yes | Yes | -| 13 | telemetryInterval | No | Yes | -| 14 | telemetryServer | No | Yes | -| 15 | telemetryPort | No | Yes | -| 16 | queryPolicy | No | Yes | -| 17 | querySmaOptimize | No | Yes | -| 18 | queryRsmaTolerance | No | Yes | -| 19 | queryBufferSize | Yes | Yes | -| 20 | maxNumOfDistinctRes | Yes | Yes | -| 21 | minSlidingTime | Yes | Yes | -| 22 | minIntervalTime | Yes | Yes | -| 23 | countAlwaysReturnValue | Yes | Yes | -| 24 | dataDir | Yes | Yes | -| 25 | minimalDataDirGB | Yes | Yes | -| 26 | supportVnodes | No | Yes | -| 27 | tempDir | Yes | Yes | -| 28 | minimalTmpDirGB | Yes | Yes | -| 29 | compressMsgSize | Yes | Yes | -| 30 | compressColData | Yes | Yes | -| 31 | smlChildTableName | Yes | Yes | -| 32 | smlTagName | Yes | Yes | -| 33 | smlDataFormat | No | Yes | -| 34 | statusInterval | Yes | Yes | -| 35 | shellActivityTimer | Yes | Yes | -| 36 | transPullupInterval | No | Yes | -| 37 | mqRebalanceInterval | No | Yes | -| 38 | ttlUnit | No | Yes | -| 39 | ttlPushInterval | No | Yes | -| 40 | numOfTaskQueueThreads | No | Yes | -| 41 | numOfRpcThreads | No | Yes | -| 42 | numOfCommitThreads | Yes | Yes | -| 43 | numOfMnodeReadThreads | No | Yes | -| 44 | numOfVnodeQueryThreads | No | Yes | -| 45 | ratioOfVnodeStreamThreads | No | Yes | -| 46 | numOfVnodeFetchThreads | No | Yes | -| 47 | numOfVnodeRsmaThreads | No | Yes | -| 48 | numOfQnodeQueryThreads | No | Yes | -| 49 | numOfQnodeFetchThreads | No | Yes | -| 50 | numOfSnodeSharedThreads | No | Yes | -| 51 | numOfSnodeUniqueThreads | No | Yes | -| 52 | rpcQueueMemoryAllowed | No | Yes | -| 53 | logDir | Yes | Yes | -| 54 | minimalLogDirGB | Yes | Yes | -| 55 | numOfLogLines | Yes | Yes | -| 56 | asyncLog | Yes | Yes | -| 57 | logKeepDays | Yes | Yes | -| 60 | debugFlag | Yes | Yes | -| 61 | tmrDebugFlag | Yes | Yes | -| 62 | uDebugFlag | Yes | Yes | -| 63 | rpcDebugFlag | Yes | Yes | -| 64 | jniDebugFlag | Yes | Yes | -| 65 | qDebugFlag | Yes | Yes | -| 66 | cDebugFlag | Yes | Yes | -| 67 | dDebugFlag | Yes | Yes | -| 68 | vDebugFlag | Yes | Yes | -| 69 | mDebugFlag | Yes | Yes | -| 70 | wDebugFlag | Yes | Yes | -| 71 | sDebugFlag | Yes | Yes | -| 72 | tsdbDebugFlag | Yes | Yes | -| 73 | tqDebugFlag | No | Yes | -| 74 | fsDebugFlag | Yes | Yes | -| 75 | udfDebugFlag | No | Yes | -| 76 | smaDebugFlag | No | Yes | -| 77 | idxDebugFlag | No | Yes | -| 78 | tdbDebugFlag | No | Yes | -| 79 | metaDebugFlag | No | Yes | -| 80 | timezone | Yes | Yes | -| 81 | locale | Yes | Yes | -| 82 | charset | Yes | Yes | -| 83 | udf | Yes | Yes | -| 84 | enableCoreFile | Yes | Yes | -| 85 | arbitrator | Yes | No | -| 86 | numOfThreadsPerCore | Yes | No | -| 87 | numOfMnodes | Yes | No | -| 88 | vnodeBak | Yes | No | -| 89 | balance | Yes | No | -| 90 | balanceInterval | Yes | No | -| 91 | offlineThreshold | Yes | No | -| 92 | role | Yes | No | -| 93 | dnodeNopLoop | Yes | No | -| 94 | keepTimeOffset | Yes | No | -| 95 | rpcTimer | Yes | No | -| 96 | rpcMaxTime | Yes | No | -| 97 | rpcForceTcp | Yes | No | -| 98 | tcpConnTimeout | Yes | No | -| 99 | syncCheckInterval | Yes | No | -| 100 | maxTmrCtrl | Yes | No | -| 101 | monitorReplica | Yes | No | -| 102 | smlTagNullName | Yes | No | -| 103 | keepColumnName | Yes | No | -| 104 | ratioOfQueryCores | Yes | No | -| 105 | maxStreamCompDelay | Yes | No | -| 106 | maxFirstStreamCompDelay | Yes | No | -| 107 | retryStreamCompDelay | Yes | No | -| 108 | streamCompDelayRatio | Yes | No | -| 109 | maxVgroupsPerDb | Yes | No | -| 110 | maxTablesPerVnode | Yes | No | -| 111 | minTablesPerVnode | Yes | No | -| 112 | tableIncStepPerVnode | Yes | No | -| 113 | cache | Yes | No | -| 114 | blocks | Yes | No | -| 115 | days | Yes | No | -| 116 | keep | Yes | No | -| 117 | minRows | Yes | No | -| 118 | maxRows | Yes | No | -| 119 | quorum | Yes | No | -| 120 | comp | Yes | No | -| 121 | walLevel | Yes | No | -| 122 | fsync | Yes | No | -| 123 | replica | Yes | No | -| 124 | partitions | Yes | No | -| 125 | quorum | Yes | No | -| 126 | update | Yes | No | -| 127 | cachelast | Yes | No | -| 128 | maxSQLLength | Yes | No | -| 129 | maxWildCardsLength | Yes | No | -| 130 | maxRegexStringLen | Yes | No | -| 131 | maxNumOfOrderedRes | Yes | No | -| 132 | maxConnections | Yes | No | -| 133 | mnodeEqualVnodeNum | Yes | No | -| 134 | http | Yes | No | -| 135 | httpEnableRecordSql | Yes | No | -| 136 | httpMaxThreads | Yes | No | -| 137 | restfulRowLimit | Yes | No | -| 138 | httpDbNameMandatory | Yes | No | -| 139 | httpKeepAlive | Yes | No | -| 140 | enableRecordSql | Yes | No | -| 141 | maxBinaryDisplayWidth | Yes | No | -| 142 | stream | Yes | No | -| 143 | retrieveBlockingModel | Yes | No | -| 144 | tsdbMetaCompactRatio | Yes | No | -| 145 | defaultJSONStrType | Yes | No | -| 146 | walFlushSize | Yes | No | -| 147 | keepTimeOffset | Yes | No | -| 148 | flowctrl | Yes | No | -| 149 | slaveQuery | Yes | No | -| 150 | adjustMaster | Yes | No | -| 151 | topicBinaryLen | Yes | No | -| 152 | telegrafUseFieldNum | Yes | No | -| 153 | deadLockKillQuery | Yes | No | -| 154 | clientMerge | Yes | No | -| 155 | sdbDebugFlag | Yes | No | -| 156 | odbcDebugFlag | Yes | No | -| 157 | httpDebugFlag | Yes | No | -| 158 | monDebugFlag | Yes | No | -| 159 | cqDebugFlag | Yes | No | -| 160 | shortcutFlag | Yes | No | -| 161 | probeSeconds | Yes | No | -| 162 | probeKillSeconds | Yes | No | -| 163 | probeInterval | Yes | No | -| 164 | lossyColumns | Yes | No | -| 165 | fPrecision | Yes | No | -| 166 | dPrecision | Yes | No | -| 167 | maxRange | Yes | No | -| 168 | range | Yes | No | + +## 3.0 Parameters + +| # | **参数** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | firstEp | Yes | Yes | | +| 2 | secondEp | Yes | Yes | | +| 3 | fqdn | Yes | Yes | | +| 4 | serverPort | Yes | Yes | | +| 5 | maxShellConns | Yes | Yes | | +| 6 | monitor | Yes | Yes | | +| 7 | monitorFqdn | No | Yes | | +| 8 | monitorPort | No | Yes | | +| 9 | monitorInterval | Yes | Yes | | +| 10 | queryPolicy | No | Yes | | +| 11 | querySmaOptimize | No | Yes | | +| 12 | maxNumOfDistinctRes | Yes | Yes | | +| 15 | countAlwaysReturnValue | Yes | Yes | | +| 16 | dataDir | Yes | Yes | | +| 17 | minimalDataDirGB | Yes | Yes | | +| 18 | supportVnodes | No | Yes | | +| 19 | tempDir | Yes | Yes | | +| 20 | minimalTmpDirGB | Yes | Yes | | +| 21 | smlChildTableName | Yes | Yes | | +| 22 | smlTagName | Yes | Yes | | +| 23 | smlDataFormat | No | Yes(discarded since 3.0.3.0) | | +| 24 | statusInterval | Yes | Yes | | +| 25 | logDir | Yes | Yes | | +| 26 | minimalLogDirGB | Yes | Yes | | +| 27 | numOfLogLines | Yes | Yes | | +| 28 | asyncLog | Yes | Yes | | +| 29 | logKeepDays | Yes | Yes | | +| 30 | debugFlag | Yes | Yes | | +| 31 | tmrDebugFlag | Yes | Yes | | +| 32 | uDebugFlag | Yes | Yes | | +| 33 | rpcDebugFlag | Yes | Yes | | +| 34 | jniDebugFlag | Yes | Yes | | +| 35 | qDebugFlag | Yes | Yes | | +| 36 | cDebugFlag | Yes | Yes | | +| 37 | dDebugFlag | Yes | Yes | | +| 38 | vDebugFlag | Yes | Yes | | +| 39 | mDebugFlag | Yes | Yes | | +| 40 | wDebugFlag | Yes | Yes | | +| 41 | sDebugFlag | Yes | Yes | | +| 42 | tsdbDebugFlag | Yes | Yes | | +| 43 | tqDebugFlag | No | Yes | | +| 44 | fsDebugFlag | Yes | Yes | | +| 45 | udfDebugFlag | No | Yes | | +| 46 | smaDebugFlag | No | Yes | | +| 47 | idxDebugFlag | No | Yes | | +| 48 | tdbDebugFlag | No | Yes | | +| 49 | metaDebugFlag | No | Yes | | +| 50 | timezone | Yes | Yes | | +| 51 | locale | Yes | Yes | | +| 52 | charset | Yes | Yes | | +| 53 | udf | Yes | Yes | | +| 54 | enableCoreFile | Yes | Yes | | diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index 19b036418f18637bfd21fa286f24528c649d146d..651892c8b29447647d8edef12af58c469de951b1 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -1,6 +1,6 @@ --- title: File directory structure -description: "TDengine installation directory description" +description: This document describes the structure of the TDengine directory after installation. --- After TDengine is installed, the following directories or files will be created in the system by default. diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 10321ab083e6e654e66cb73f1bc21f9fbd678fda..caedd76df8e41b4246891f686d61ed61af671d7e 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -1,6 +1,6 @@ --- title: Schemaless Writing -description: 'The Schemaless write method eliminates the need to create super tables/sub tables in advance and automatically creates the storage structure corresponding to the data, as it is written to the interface.' +description: This document describes how to use the schemaless write component of TDengine. --- In IoT applications, data is collected for many purposes such as intelligent control, business analysis, device monitoring and so on. Due to changes in business or functional requirements or changes in device hardware, the application logic and even the data collected may change. Schemaless writing automatically creates storage structures for your data as it is being written to TDengine, so that you do not need to create supertables in advance. When necessary, schemaless writing @@ -80,7 +80,7 @@ You can configure smlChildTableName in taos.cfg to specify table names, for exam NULL. 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. 7. Errors encountered throughout the processing will interrupt the writing process and return an error code. -8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) +8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, discarded since 3.0.3.0) :::tip All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 665bc75380d4f59666d792d074fb37c65c810264..16c591154705521fc194a13ddcf6d2bbb5552470 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -1,7 +1,7 @@ --- sidebar_label: taosKeeper title: taosKeeper -description: exports TDengine monitoring metrics. +description: This document describes how to use taosKeeper, a tool for exporting TDengine monitoring metrics. --- ## Introduction @@ -24,7 +24,14 @@ You can compile taosKeeper separately and install it. Please refer to the [taosK taosKeeper needs to be executed on the terminal of the operating system, it supports three configuration methods: [Command-line arguments](#command-line-arguments-in-detail), [environment variable](#environment-variable-in-detail) and [configuration file](#configuration-file-parameters-in-detail). The precedence of those is Command-line, environment variable and configuration file. -**Make sure that the TDengine cluster is running correctly before running taosKeeper. ** Ensure that the monitoring service in TDengine has been started. For more information, see [TDengine Monitoring Configuration](../config/#monitoring). +**Make sure that the TDengine cluster is running correctly before running taosKeeper.** Ensure that the monitoring service in TDengine has been started. At least the values of `monitor` and `monitorFqdn` need to be set in `taos.cfg`. + +```shell +monitor 1 +monitorFqdn localhost # taoskeeper's FQDN +``` + +For more information, see [TDengine Monitoring Configuration](../config/#monitoring). ### Command-Line Parameters diff --git a/docs/en/14-reference/index.md b/docs/en/14-reference/index.md index f3a64913d065d1d8e321ce7433c9d605ef70bd13..bc8ec6996538bf15bf4cb4d52aaa126ed11b5b72 100644 --- a/docs/en/14-reference/index.md +++ b/docs/en/14-reference/index.md @@ -1,5 +1,6 @@ --- title: Reference +description: This document describes TDengine connectors and utilities. --- This section describes the TDengine connectors and utilities. diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index ca32ce8afce30eaa6756f7b403685b989d824bbf..5a2942b14497401ef1a2cb8f31ad2773f6b828e3 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -1,6 +1,7 @@ --- -sidebar_label: Grafana title: Grafana +sidebar_label: Grafana +description: This document describes how to integrate TDengine with Grafana. --- import Tabs from "@theme/Tabs"; @@ -155,13 +156,13 @@ You can setup a zero-configuration stack for TDengine + Grafana by [docker-compo services: tdengine: - image: tdengine/tdengine:2.6.0.2 + image: tdengine/tdengine:3.0.2.4 environment: TAOS_FQDN: tdengine volumes: - tdengine-data:/var/lib/taos/ grafana: - image: grafana/grafana:8.5.6 + image: grafana/grafana:9.3.6 volumes: - ./tdengine.yml/:/etc/grafana/provisioning/tdengine.yml - grafana-data:/var/lib/grafana @@ -196,11 +197,18 @@ As shown above, select the `TDengine` data source in the `Query` and enter the c - INPUT SQL: Enter the desired query (the results being two columns and multiple rows), such as `select _wstart, avg(mem_system) from log.dnodes_info where ts >= $from and ts < $to interval($interval)`. In this statement, $from, $to, and $interval are variables that Grafana replaces with the query time range and interval. In addition to the built-in variables, custom template variables are also supported. - ALIAS BY: This allows you to set the current query alias. - GENERATE SQL: Clicking this button will automatically replace the corresponding variables and generate the final executed statement. +- Group by column name(s): `group by` or `partition by` columns name split by comma. By setting `Group by column name(s)`, it can show multi-dimension data if Sql is `group by` or `partition by`. Such as, it can show data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep`. +- Format to: format legend for `group by` or `partition by`. Such as it can display series data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep` and `Format to` is `mem_system_{{dnode_ep}}`. Follow the default prompt to query the average system memory usage for the specified interval on the server where the current TDengine deployment is located as follows. ![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard2.webp) +查询每台 TDengine 服务器指定间隔系统内存平均使用量如下. +The example to query the average system memory usage for the specified interval on each server as follows. + +![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard3.webp) + > For more information on how to use Grafana to create the appropriate monitoring interface and for more details on using Grafana, refer to the official Grafana [documentation](https://grafana.com/docs/). ### Importing the Dashboard diff --git a/docs/en/20-third-party/02-prometheus.md b/docs/en/20-third-party/02-prometheus.md index ef9b9cb637ffc8da2de6acdd853fa097cfbc0193..bfdd3d015e6fcfa484fdceea6f0bebc43049ed17 100644 --- a/docs/en/20-third-party/02-prometheus.md +++ b/docs/en/20-third-party/02-prometheus.md @@ -1,6 +1,7 @@ --- -sidebar_label: Prometheus title: Prometheus writing and reading +sidebar_label: Prometheus +description: This document describes how to integrate TDengine with Prometheus. --- import Prometheus from "../14-reference/_prometheus.mdx" diff --git a/docs/en/20-third-party/03-telegraf.md b/docs/en/20-third-party/03-telegraf.md index 8f3cab0e57094a1d2fac5f54601489382ac3f9a1..7e99b84eab88614bf82da92a2db11ddf1be753bf 100644 --- a/docs/en/20-third-party/03-telegraf.md +++ b/docs/en/20-third-party/03-telegraf.md @@ -1,6 +1,7 @@ --- -sidebar_label: Telegraf title: Telegraf writing +sidebar_label: Telegraf +description: This document describes how to integrate TDengine with Telegraf. --- import Telegraf from "../14-reference/_telegraf.mdx" diff --git a/docs/en/20-third-party/05-collectd.md b/docs/en/20-third-party/05-collectd.md index 5b52e3b7bc75c2d21ab5a6c08331030fe4e423f7..d8c8e7f81def389418d28d62486ef32afb5d996c 100644 --- a/docs/en/20-third-party/05-collectd.md +++ b/docs/en/20-third-party/05-collectd.md @@ -1,6 +1,7 @@ --- -sidebar_label: collectd title: collectd writing +sidebar_label: collectd +description: This document describes how to integrate TDengine with collectd. --- import CollectD from "../14-reference/_collectd.mdx" diff --git a/docs/en/20-third-party/06-statsd.md b/docs/en/20-third-party/06-statsd.md index b861a48ecd59fa1f2036c6c323d3fdee8a3ee44b..ea428e9cdb45992cdebd6be4713583b88990a67f 100644 --- a/docs/en/20-third-party/06-statsd.md +++ b/docs/en/20-third-party/06-statsd.md @@ -1,6 +1,7 @@ --- -sidebar_label: StatsD title: StatsD Writing +sidebar_label: StatsD +description: This document describes how to integrate TDengine with StatsD. --- import StatsD from "../14-reference/_statsd.mdx" diff --git a/docs/en/20-third-party/07-icinga2.md b/docs/en/20-third-party/07-icinga2.md index 167b6a4303f838c8c9945a95dac802a54bad3525..540aae8689bc0885ab4649be8fd07fec04dad829 100644 --- a/docs/en/20-third-party/07-icinga2.md +++ b/docs/en/20-third-party/07-icinga2.md @@ -1,6 +1,7 @@ --- -sidebar_label: icinga2 title: icinga2 writing +sidebar_label: icinga2 +description: This document describes how to integrate TDengine with icinga2. --- import Icinga2 from "../14-reference/_icinga2.mdx" diff --git a/docs/en/20-third-party/08-tcollector.md b/docs/en/20-third-party/08-tcollector.md index b604a2d7125851a8f2ec4965f755f5400d0613bf..f1c0ecd44deaea6172d8d9b1d4e6c2e5ee8ccabb 100644 --- a/docs/en/20-third-party/08-tcollector.md +++ b/docs/en/20-third-party/08-tcollector.md @@ -1,6 +1,7 @@ --- -sidebar_label: TCollector title: TCollector writing +sidebar_label: TCollector +description: This document describes how to integrate TDengine with TCollector. --- import TCollector from "../14-reference/_tcollector.mdx" diff --git a/docs/en/20-third-party/09-emq-broker.md b/docs/en/20-third-party/09-emq-broker.md index 2ead1bbaf40f06fec2a5cbf85e46fdfdcc5216df..10ce0174ed5d362a113acd7ade59741a10155657 100644 --- a/docs/en/20-third-party/09-emq-broker.md +++ b/docs/en/20-third-party/09-emq-broker.md @@ -1,6 +1,7 @@ --- -sidebar_label: EMQX Broker title: EMQX Broker writing +sidebar_label: EMQX Broker +description: This document describes how to integrate TDengine with the EMQX broker. --- MQTT is a popular IoT data transfer protocol. [EMQX](https://github.com/emqx/emqx) is an open-source MQTT Broker software. You can write MQTT data directly to TDengine without any code. You only need to setup "rules" in EMQX Dashboard to create a simple configuration. EMQX supports saving data to TDengine by sending data to a web service and provides a native TDengine driver for direct saving in the Enterprise Edition. Please refer to the [EMQX official documentation](https://www.emqx.io/docs/en/v4.4/rule/rule-engine.html) for details on how to use it.). diff --git a/docs/en/20-third-party/10-hive-mq-broker.md b/docs/en/20-third-party/10-hive-mq-broker.md index 828a62ac5b336766d5c3770cc42cd3a61cfd8d5d..6c5165596e04678db1dd32d16756133719fdb4b9 100644 --- a/docs/en/20-third-party/10-hive-mq-broker.md +++ b/docs/en/20-third-party/10-hive-mq-broker.md @@ -1,6 +1,7 @@ --- -sidebar_label: HiveMQ Broker title: HiveMQ Broker Writing +sidebar_label: HiveMQ Broker +description: This document describes how to integrate TDengine with the HiveMQ broker. --- [HiveMQ](https://www.hivemq.com/) is an MQTT broker that provides community and enterprise editions. HiveMQ is mainly for enterprise emerging machine-to-machine M2M communication and internal transport, meeting scalability, ease of management, and security features. HiveMQ provides an open-source plug-in development kit. MQTT data can be saved to TDengine via TDengine extension for HiveMQ. For more information, see [HiveMQ TDengine Extension](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README_EN.md). diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 3e8f7c295d7b6ab81ecb678865ed3e4eff81b2c5..0998862b2d75b98fb65ff414d000535b0c8b7ea5 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -1,6 +1,7 @@ --- -sidebar_label: Kafka title: TDengine Kafka Connector Tutorial +sidebar_label: Kafka +description: This document describes how to integrate TDengine with Kafka. --- TDengine Kafka Connector contains two plugins: TDengine Source Connector and TDengine Sink Connector. Users only need to provide a simple configuration file to synchronize the data of the specified topic in Kafka (batch or real-time) to TDengine or synchronize the data (batch or real-time) of the specified database in TDengine to Kafka. diff --git a/docs/en/20-third-party/12-google-data-studio.md b/docs/en/20-third-party/12-google-data-studio.md index fc94f98056bbeeeec88ca7ea12a4a6a7e6f15dc5..ea6431fa5a81b3b9e51fe04e8e404cf52e9c0ad9 100644 --- a/docs/en/20-third-party/12-google-data-studio.md +++ b/docs/en/20-third-party/12-google-data-studio.md @@ -1,6 +1,7 @@ --- -sidebar_label: Google Data Studio title: Use Google Data Studio to access TDengine +sidebar_label: Google Data Studio +description: This document describes how to integrate TDengine with Google Data Studio. --- Data Studio is a powerful tool for reporting and visualization, offering a wide variety of charts and connectors and making it easy to generate reports based on predefined templates. Its ease of use and robust ecosystem have made it one of the first choices for people working in data analysis. diff --git a/docs/en/20-third-party/13-Jupyter.md b/docs/en/20-third-party/13-Jupyter.md index fbd7e530f0959740c53e48ce1d73d92ce0d6c5c5..1ac9df1da4c7756c4f186d456948ae65262e9d54 100644 --- a/docs/en/20-third-party/13-Jupyter.md +++ b/docs/en/20-third-party/13-Jupyter.md @@ -1,6 +1,7 @@ --- -sidebar_label: JupyterLab title: Connect JupyterLab to TDengine +sidebar_label: JupyterLab +description: This document describes how to integrate TDengine with JupyterLab. --- JupyterLab is the next generation of the ubiquitous Jupyter Notebook. In this note we show you how to install the TDengine Python connector to connect to TDengine in JupyterLab. You can then insert data and perform queries against the TDengine instance within JupyterLab. diff --git a/docs/en/20-third-party/grafana/create_dashboard1.webp b/docs/en/20-third-party/grafana/create_dashboard1.webp index 55eb388833e4df2a46f4d1cf6d346aa11429385d..df41d66d50e888555792ef34188a966c2e106a86 100644 Binary files a/docs/en/20-third-party/grafana/create_dashboard1.webp and b/docs/en/20-third-party/grafana/create_dashboard1.webp differ diff --git a/docs/en/20-third-party/grafana/create_dashboard2.webp b/docs/en/20-third-party/grafana/create_dashboard2.webp index bb40e407187718c52e9f617d8ebd3d25fd14b56b..ba50f5299605bd72ebfe1f3f5d1d62bdbc8939c9 100644 Binary files a/docs/en/20-third-party/grafana/create_dashboard2.webp and b/docs/en/20-third-party/grafana/create_dashboard2.webp differ diff --git a/docs/en/20-third-party/grafana/create_dashboard3.webp b/docs/en/20-third-party/grafana/create_dashboard3.webp new file mode 100644 index 0000000000000000000000000000000000000000..84f387f87bcc8e06fca90deabebd1d601c528ef1 Binary files /dev/null and b/docs/en/20-third-party/grafana/create_dashboard3.webp differ diff --git a/docs/en/20-third-party/index.md b/docs/en/20-third-party/index.md index 87bd9e075133d1182ee93d1c1c43617c766755b9..6fc8043eefef62d476e289195c8f38e385e020c4 100644 --- a/docs/en/20-third-party/index.md +++ b/docs/en/20-third-party/index.md @@ -1,5 +1,6 @@ --- title: Third Party Tools +description: This document describes how to integrate TDengine with various third-party tools. --- Since TDengine supports standard SQL commands, common database connector standards (e.g., JDBC), ORM, and other popular time-series database writing protocols (e.g., InfluxDB Line Protocol, OpenTSDB JSON, OpenTSDB Telnet, etc.), it is very easy to integrate TDengine with other third party tools. You only need to provide simple configuration, the integration can be done without a line of code. diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index 697ecb98a1c212dbdfef47a916fe1e288ea5508c..cef05dcc56793846fd36515121f6688f74999bbf 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -1,6 +1,7 @@ --- -sidebar_label: Architecture title: Architecture +sidebar_label: Architecture +description: This document describes the architecture of TDengine. --- ## Cluster and Primary Logic Unit diff --git a/docs/en/21-tdinternal/03-high-availability.md b/docs/en/21-tdinternal/03-high-availability.md index e2e1c6521efd59078e7dc9c9a1e5d8df3377d266..a0f6ca4ffe15ba5901709ffa3c6d64b8697fa93a 100644 --- a/docs/en/21-tdinternal/03-high-availability.md +++ b/docs/en/21-tdinternal/03-high-availability.md @@ -1,6 +1,7 @@ --- -sidebar_label: High Availability title: High Availability +sidebar_label: High Availability +description: This document describes how TDengine implements high availability. --- ## High Availability of Vnode diff --git a/docs/en/21-tdinternal/04-load-balance.md b/docs/en/21-tdinternal/04-load-balance.md index 7648398059a2a422533d6d687336988d207c79a4..73da1c1dd65ffb3251c86ff936f5b5b26dc9d1cb 100644 --- a/docs/en/21-tdinternal/04-load-balance.md +++ b/docs/en/21-tdinternal/04-load-balance.md @@ -1,6 +1,7 @@ --- -sidebar_label: Load Balance title: Load Balance +sidebar_label: Load Balance +description: This document describes how TDengine implements load balancing. --- The load balance in TDengine is mainly about processing data series data. TDengine employes builtin hash algorithm to distribute all the tables, sub-tables and their data of a database across all the vgroups that belongs to the database. Each table or sub-table can only be handled by a single vgroup, while each vgroup can process multiple table or sub-table. diff --git a/docs/en/21-tdinternal/index.md b/docs/en/21-tdinternal/index.md index 999d6f89ff57164d6e0372620504c8ecc9de7c9b..4f0846754382767b4694ab1c54f8ae2d3340948c 100644 --- a/docs/en/21-tdinternal/index.md +++ b/docs/en/21-tdinternal/index.md @@ -1,5 +1,6 @@ --- title: TDengine Inside +description: This document describes the internals of TDengine from an architectural perspective. --- ```mdx-code-block diff --git a/docs/en/25-application/01-telegraf.md b/docs/en/25-application/01-telegraf.md index 65fb08ee674cc9c952bbcfda57f0366fa129fdf2..e043aebcb64f6e3b26b1e79964c5875e3b64ee6f 100644 --- a/docs/en/25-application/01-telegraf.md +++ b/docs/en/25-application/01-telegraf.md @@ -1,6 +1,7 @@ --- -sidebar_label: TDengine + Telegraf + Grafana title: Quickly Build IT DevOps Visualization System with TDengine + Telegraf + Grafana +sidebar_label: TDengine + Telegraf + Grafana +description: This document describes how to create an IT visualization system by integrating TDengine with Telegraf and Grafana. --- ## Background diff --git a/docs/en/25-application/02-collectd.md b/docs/en/25-application/02-collectd.md index 97412b230941435a08df12c859d63bca68b79ec9..6ac7253fc46c50d3efe698d508de8669c65bf5d9 100644 --- a/docs/en/25-application/02-collectd.md +++ b/docs/en/25-application/02-collectd.md @@ -1,6 +1,7 @@ --- -sidebar_label: TDengine + collectd/StatsD + Grafana title: Quickly build an IT DevOps visualization system using TDengine + collectd/StatsD + Grafana +sidebar_label: TDengine + collectd/StatsD + Grafana +description: This document describes how to build an IT visualization system by integrating TDengine with Grafana and collectd or StatsD. --- ## Background diff --git a/docs/en/25-application/03-immigrate.md b/docs/en/25-application/_03-immigrate.md similarity index 99% rename from docs/en/25-application/03-immigrate.md rename to docs/en/25-application/_03-immigrate.md index 1aabaa43e77660d72bca00d7d59cdee69b1a7c92..5f4a86937e305c517bca36f6fcf3be6ae791dd20 100644 --- a/docs/en/25-application/03-immigrate.md +++ b/docs/en/25-application/_03-immigrate.md @@ -1,6 +1,7 @@ --- -sidebar_label: OpenTSDB Migration to TDengine title: Best Practices for Migrating OpenTSDB Applications to TDengine +sidebar_label: OpenTSDB Migration to TDengine +description: This document describes the best practices for migrating an OpenTSDB application to TDengine. --- As a distributed, scalable, distributed time-series database platform based on HBase, and thanks to its first-mover advantage, OpenTSDB is widely used for monitoring in DevOps. However, as new technologies like cloud computing, microservices, and containerization technology has developed rapidly, Enterprise-level services are becoming more and more diverse and the architecture is becoming more complex. @@ -183,7 +184,7 @@ TDengine supports the standard JDBC 3.0 interface for manipulating databases, bu To facilitate historical data migration, we provide a plug-in for the data synchronization tool DataX, which can automatically write data into TDengine.The automatic data migration of DataX can only support the data migration process of a single value model. -For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/blog/2021/10/26/3156.html). +For the specific usage of DataX and how to use DataX to write data to TDengine, please refer to [DataX-based TDengine Data Migration Tool](https://www.taosdata.com/engineering/16401.html). After migrating via DataX, we found that we can significantly improve the efficiency of migrating historical data by starting multiple processes and migrating numerous metrics simultaneously. The following are some records of the migration process. We provide these as a reference for application migration. diff --git a/docs/en/25-application/index.md b/docs/en/25-application/index.md index 5383a00c67c515dc65fd2ed1cac4b218710288b5..178fef47b64285efe732e53b316581ae4876a218 100644 --- a/docs/en/25-application/index.md +++ b/docs/en/25-application/index.md @@ -1,5 +1,6 @@ --- title: Application Practice +description: This document describes some examples of building systems around TDengine. --- ```mdx-code-block diff --git a/docs/en/27-train-faq/01-faq.md b/docs/en/27-train-faq/01-faq.md index 82e98b0d980c16acad0783abd62525cc6bde06ec..9e7f4f8e0d3d94f0016c0dcc6701c670fd7d32c8 100644 --- a/docs/en/27-train-faq/01-faq.md +++ b/docs/en/27-train-faq/01-faq.md @@ -1,5 +1,6 @@ --- title: Frequently Asked Questions +description: This document describes the frequently asked questions about TDengine. --- ## Submit an Issue @@ -33,7 +34,7 @@ TDengine 3.0 is not compatible with the configuration and data files from previo 4. Install TDengine 3.0. 5. For assistance in migrating data to TDengine 3.0, contact [TDengine Support](https://tdengine.com/support). -### 4. How can I resolve the "Unable to establish connection" error? +### 2. How can I resolve the "Unable to establish connection" error? This error indicates that the client could not connect to the server. Perform the following troubleshooting steps: @@ -68,7 +69,7 @@ This error indicates that the client could not connect to the server. Perform th 11. You can also use the TDengine CLI to diagnose network issues. For more information, see [Problem Diagnostics](https://docs.tdengine.com/operation/diagnose/). -### 5. How can I resolve the "Unable to resolve FQDN" error? +### 3. How can I resolve the "Unable to resolve FQDN" error? Clients and dnodes must be able to resolve the FQDN of each required node. You can confirm your configuration as follows: @@ -79,15 +80,15 @@ Clients and dnodes must be able to resolve the FQDN of each required node. You c 5. If TDengine has been previously installed and the `hostname` was modified, open `dnode.json` in the `data` folder and verify that the endpoint configuration is correct. The default location of the dnode file is `/var/lib/taos/dnode`. Ensure that you clean up previous installations before reinstalling TDengine. 6. Confirm whether FQDNs are preconfigured in `/etc/hosts` and `/etc/hostname`. -### 6. What is the most effective way to write data to TDengine? +### 4. What is the most effective way to write data to TDengine? Writing data in batches provides higher efficiency in most situations. You can insert one or more data records into one or more tables in a single SQL statement. -### 9. Why are table names not fully displayed? +### 5. Why are table names not fully displayed? The number of columns in the TDengine CLI terminal display is limited. This can cause table names to be cut off, and if you use an incomplete name in a statement, the "Table does not exist" error will occur. You can increase the display size with the `maxBinaryDisplayWidth` parameter or the SQL statement `set max_binary_display_width`. You can also append `\G` to your SQL statement to bypass this limitation. -### 10. How can I migrate data? +### 6. How can I migrate data? In TDengine, the `hostname` uniquely identifies a machine. When you move data files to a new machine, you must configure the new machine to have the same `host name` as the original machine. @@ -97,7 +98,7 @@ The data structure of previous versions of TDengine is not compatible with versi ::: -### 11. How can I temporary change the log level from the TDengine Client? +### 7. How can I temporary change the log level from the TDengine Client? To change the log level for debugging purposes, you can use the following command: @@ -118,14 +119,14 @@ Use `resetlog` to remove all logs generated on the local client. Use the other p For each parameter, you can set the value to `131` (error and warning), `135` (error, warning, and debug), or `143` (error, warning, debug, and trace). -### Why do TDengine components written in Go fail to compile? +### 8. Why do TDengine components written in Go fail to compile? TDengine includes taosAdapter, an independent component written in Go. This component provides the REST API as well as data access for other products such as Prometheus and Telegraf. When using the develop branch, you must run `git submodule update --init --recursive` to download the taosAdapter repository and then compile it. TDengine Go components require Go version 1.14 or later. -### 13. How can I query the storage space being used by my data? +### 9. How can I query the storage space being used by my data? The TDengine data files are stored in `/var/lib/taos` by default. Log files are stored in `/var/log/taos`. @@ -133,7 +134,7 @@ To see how much space your data files occupy, run `du -sh /var/lib/taos/vnode -- If you want to see how much space is occupied by a single database, first determine which vgroup is storing the database by running `show vgroups`. Then check `/var/lib/taos/vnode` for the files associated with the vgroup ID. -### 15. How is timezone information processed for timestamps? +### 10. How is timezone information processed for timestamps? TDengine uses the timezone of the client for timestamps. The server timezone does not affect timestamps. The client converts Unix timestamps in SQL statements to UTC before sending them to the server. When you query data on the server, it provides timestamps in UTC to the client, which converts them to its local time. @@ -144,13 +145,13 @@ Timestamps are processed as follows: 3. A timezone explicitly specified when establishing a connection to TDengine through a connector takes precedence over `taos.cfg` and the system timezone. For example, the Java connector allows you to specify a timezone in the JDBC URL. 4. If you use an RFC 3339 timestamp (2013-04-12T15:52:01.123+08:00), or an ISO 8601 timestamp (2013-04-12T15:52:01.123+0800), the timezone specified in the timestamp is used instead of the timestamps configured using any other method. -### 16. Which network ports are required by TDengine? +### 11. Which network ports are required by TDengine? See [serverPort](https://docs.tdengine.com/reference/config/#serverport) in Configuration Parameters. Note that ports are specified using 6030 as the default first port. If you change this port, all other ports change as well. -### 17. Why do applications such as Grafana fail to connect to TDengine over the REST API? +### 12. Why do applications such as Grafana fail to connect to TDengine over the REST API? In TDengine, the REST API is provided by taosAdapter. Ensure that taosAdapter is running before you connect an application to TDengine over the REST API. You can run `systemctl start taosadapter` to start the service. @@ -158,7 +159,7 @@ Note that the log path for taosAdapter must be configured separately. The defaul For more information, see [taosAdapter](https://docs.tdengine.com/reference/taosadapter/). -### 18. How can I resolve out-of-memory (OOM) errors? +### 13. How can I resolve out-of-memory (OOM) errors? OOM errors are thrown by the operating system when its memory, including swap, becomes insufficient and it needs to terminate processes to remain operational. Most OOM errors in TDengine occur for one of the following reasons: free memory is less than the value of `vm.min_free_kbytes` or free memory is less than the size of the request. If TDengine occupies reserved memory, an OOM error can occur even when free memory is sufficient. diff --git a/docs/en/27-train-faq/index.md b/docs/en/27-train-faq/index.md index 2cb87aab005b0ecc9275b6fe10e267487d38c336..cc55a41559a32fcb89b28a99705b59e07c22a81d 100644 --- a/docs/en/27-train-faq/index.md +++ b/docs/en/27-train-faq/index.md @@ -1,5 +1,6 @@ --- title: FAQ & Others +description: This document describes common issues related with TDengine. --- ```mdx-code-block diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index 32bdc21e7c0552f5f891b38f806e48efb6c419ac..6a6210806274d5b6e2e8efe12d281e908d2d0c96 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -1,7 +1,7 @@ --- -sidebar_label: TDengine title: TDengine Release History and Download Links -description: TDengine release history, Release Notes and download links. +sidebar_label: TDengine +description: This document provides download links for all released versions of TDengine 3.0. --- TDengine 3.x installation packages can be downloaded at the following links: @@ -10,6 +10,34 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.2.6 + + + +## 3.0.2.5 + + + +## 3.0.2.4 + + + +## 3.0.2.3 + + + +## 3.0.2.2 + + + +## 3.0.2.1 + + + +## 3.0.2.0 + + + ## 3.0.1.8 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 7126b5a997043231d1cf93d633b8cd71e5f6275e..91eb0c9b8d1ee5ed3b0e774c3d2d78e3bd4c03ba 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -1,7 +1,7 @@ --- -sidebar_label: taosTools title: taosTools Release History and Download Links -description: taosTools release history, Release Notes, download links. +sidebar_label: taosTools +description: This document provides download links for all released versions of taosTools compatible with TDengine 3.0. --- taosTools installation packages can be downloaded at the following links: @@ -10,6 +10,34 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.6 + + + +## 2.4.3 + + + +## 2.4.2 + + + +## 2.4.1 + + + +## 2.4.0 + + + +## 2.3.3 + + + +## 2.3.2 + + + ## 2.3.0 diff --git a/docs/en/28-releases/index.md b/docs/en/28-releases/index.md index c01c99cdce0190fb04f88d55a09e8cc406d4d8b0..d1f93c3b9b2cf297578953d9c262b60f3ad2a214 100644 --- a/docs/en/28-releases/index.md +++ b/docs/en/28-releases/index.md @@ -1,5 +1,6 @@ --- title: Releases +description: This document describes TDengine products that have been released. --- ```mdx-code-block diff --git a/docs/examples/csharp/asyncQuery/asyncquery.csproj b/docs/examples/csharp/asyncQuery/asyncquery.csproj index 23e590cd25aa88e58cabf81717a6baf320f447bc..7c5b693f28dfa8832ae08bbaae9aa8a367951c70 100644 --- a/docs/examples/csharp/asyncQuery/asyncquery.csproj +++ b/docs/examples/csharp/asyncQuery/asyncquery.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/connect/connect.csproj b/docs/examples/csharp/connect/connect.csproj index 3a912f8987ace6ae540726886d901c8d32a7b81b..a08e86d4b42199be44a6551e37da11efb6e06a34 100644 --- a/docs/examples/csharp/connect/connect.csproj +++ b/docs/examples/csharp/connect/connect.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/influxdbLine/influxdbline.csproj b/docs/examples/csharp/influxdbLine/influxdbline.csproj index 58bca485088e409fe1d387c6020418bbc2bf871b..4889f8fde9dc0eb75c0547e32355929d1cceb138 100644 --- a/docs/examples/csharp/influxdbLine/influxdbline.csproj +++ b/docs/examples/csharp/influxdbLine/influxdbline.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/optsJSON/optsJSON.csproj b/docs/examples/csharp/optsJSON/optsJSON.csproj index da16025dcd45f8e5c4ba6e242524c2e56191e93c..208f04c82d19f83f2746871b64a6dfdf0dcf3eae 100644 --- a/docs/examples/csharp/optsJSON/optsJSON.csproj +++ b/docs/examples/csharp/optsJSON/optsJSON.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/optsTelnet/optstelnet.csproj b/docs/examples/csharp/optsTelnet/optstelnet.csproj index 194de21bcc74653a2267b29681ece6243fd401fc..32c76ec4184b82e943897a36bc3bcbbd9ec85149 100644 --- a/docs/examples/csharp/optsTelnet/optstelnet.csproj +++ b/docs/examples/csharp/optsTelnet/optstelnet.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/query/query.csproj b/docs/examples/csharp/query/query.csproj index c97dbd3051e1a415b192e73d6753266b0b41b07d..360d73b2c096ef86df59876d0629fd0c4b6a239b 100644 --- a/docs/examples/csharp/query/query.csproj +++ b/docs/examples/csharp/query/query.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/sqlInsert/sqlinsert.csproj b/docs/examples/csharp/sqlInsert/sqlinsert.csproj index ab0e5e717a78faad07c949b434b0d0b8a26c7211..1b6f745c82437e9796da4c48fc720600dbe99cb5 100644 --- a/docs/examples/csharp/sqlInsert/sqlinsert.csproj +++ b/docs/examples/csharp/sqlInsert/sqlinsert.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/stmtInsert/Program.cs b/docs/examples/csharp/stmtInsert/Program.cs index 87e1971feb8499c515206f05a1e916070ac57f4c..80cadb2ff8b596a0484d05ff15aeaa50f22ff859 100644 --- a/docs/examples/csharp/stmtInsert/Program.cs +++ b/docs/examples/csharp/stmtInsert/Program.cs @@ -42,7 +42,7 @@ namespace TDengineExample // 5. execute res = TDengine.StmtExecute(stmt); - CheckStmtRes(res, "faild to execute"); + CheckStmtRes(res, "failed to execute"); // 6. free TaosMultiBind.FreeTaosBind(tags); @@ -92,7 +92,7 @@ namespace TDengineExample int code = TDengine.StmtClose(stmt); if (code != 0) { - throw new Exception($"falied to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); + throw new Exception($"failed to close stmt, {code} reason: {TDengine.StmtErrorStr(stmt)} "); } } } diff --git a/docs/examples/csharp/stmtInsert/stmtinsert.csproj b/docs/examples/csharp/stmtInsert/stmtinsert.csproj index 3d459fbeda02ab03dc40dac2ecae290724cccbcc..f5b2b673971c3822e6f6c9b65b8f02bc9d4dc80e 100644 --- a/docs/examples/csharp/stmtInsert/stmtinsert.csproj +++ b/docs/examples/csharp/stmtInsert/stmtinsert.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/subscribe/subscribe.csproj b/docs/examples/csharp/subscribe/subscribe.csproj index 8ae1cf6bc6023558c28797a0d9fcccb2f2e87653..191b3f9e9bb07dc72c9bb452ad19e30e42af922a 100644 --- a/docs/examples/csharp/subscribe/subscribe.csproj +++ b/docs/examples/csharp/subscribe/subscribe.csproj @@ -9,7 +9,7 @@ - + diff --git a/docs/examples/csharp/wsConnect/wsConnect.csproj b/docs/examples/csharp/wsConnect/wsConnect.csproj index 34951dc761903e5a4b7a4bec5dfe55a965ab88be..c7988b6e9c955f21d53a312e10b513403363936e 100644 --- a/docs/examples/csharp/wsConnect/wsConnect.csproj +++ b/docs/examples/csharp/wsConnect/wsConnect.csproj @@ -2,12 +2,17 @@ Exe - net5.0 - enable + net6.0 - + + + + + + + diff --git a/docs/examples/csharp/wsInsert/wsInsert.csproj b/docs/examples/csharp/wsInsert/wsInsert.csproj index 34951dc761903e5a4b7a4bec5dfe55a965ab88be..5aa419b2c80bf303e84ff3f1a30a839ce9220663 100644 --- a/docs/examples/csharp/wsInsert/wsInsert.csproj +++ b/docs/examples/csharp/wsInsert/wsInsert.csproj @@ -2,12 +2,16 @@ Exe - net5.0 + net6.0 enable - - + - + + + + + + diff --git a/docs/examples/csharp/wsQuery/wsQuery.csproj b/docs/examples/csharp/wsQuery/wsQuery.csproj index 34951dc761903e5a4b7a4bec5dfe55a965ab88be..bcc7c19a59fdab1c8ad9d3098824239d645743e6 100644 --- a/docs/examples/csharp/wsQuery/wsQuery.csproj +++ b/docs/examples/csharp/wsQuery/wsQuery.csproj @@ -2,12 +2,18 @@ Exe - net5.0 + net6.0 enable - + + + + + + + diff --git a/docs/examples/csharp/wsStmt/wsStmt.csproj b/docs/examples/csharp/wsStmt/wsStmt.csproj index 34951dc761903e5a4b7a4bec5dfe55a965ab88be..bcc7c19a59fdab1c8ad9d3098824239d645743e6 100644 --- a/docs/examples/csharp/wsStmt/wsStmt.csproj +++ b/docs/examples/csharp/wsStmt/wsStmt.csproj @@ -2,12 +2,18 @@ Exe - net5.0 + net6.0 enable - + + + + + + + diff --git a/docs/examples/go/go.sum b/docs/examples/go/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..13e13adaa189053696320a6eb9740daa319a98b7 --- /dev/null +++ b/docs/examples/go/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/taosdata/driver-go/v3 v3.1.0/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 7721eed1346f254e104bb41a2135171ebd518e2c..1f7218936fbe457615562ded1b938daca95225cb 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -1,17 +1,12 @@ package main import ( - "context" - "encoding/json" "fmt" - "strconv" - "time" + "os" "github.com/taosdata/driver-go/v3/af" "github.com/taosdata/driver-go/v3/af/tmq" - "github.com/taosdata/driver-go/v3/common" - "github.com/taosdata/driver-go/v3/errors" - "github.com/taosdata/driver-go/v3/wrapper" + tmqcommon "github.com/taosdata/driver-go/v3/common/tmq" ) func main() { @@ -24,97 +19,60 @@ func main() { if err != nil { panic(err) } - _, err = db.Exec("create topic if not exists example_tmq_topic with meta as DATABASE example_tmq") + _, err = db.Exec("create topic if not exists example_tmq_topic as DATABASE example_tmq") if err != nil { panic(err) } - config := tmq.NewConfig() - defer config.Destroy() - err = config.SetGroupID("test") if err != nil { panic(err) } - err = config.SetAutoOffsetReset("earliest") - if err != nil { - panic(err) - } - err = config.SetConnectIP("127.0.0.1") - if err != nil { - panic(err) - } - err = config.SetConnectUser("root") - if err != nil { - panic(err) - } - err = config.SetConnectPass("taosdata") + consumer, err := tmq.NewConsumer(&tmqcommon.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_client", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", + }) if err != nil { panic(err) } - err = config.SetConnectPort("6030") + err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } - err = config.SetMsgWithTableName(true) + _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") if err != nil { panic(err) } - err = config.EnableHeartBeat() + _, err = db.Exec("insert into example_tmq.t1 values(now,1)") if err != nil { panic(err) } - err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) + for i := 0; i < 5; i++ { + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.String()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - }) - if err != nil { - panic(err) } - consumer, err := tmq.NewConsumer(config) + err = consumer.Unsubscribe() if err != nil { panic(err) } - err = consumer.Subscribe([]string{"example_tmq_topic"}) + err = consumer.Close() if err != nil { panic(err) } - _, err = db.Exec("create table example_tmq.t1 (ts timestamp,v int)") - if err != nil { - panic(err) - } - for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) - } - if result.Type != common.TMQ_RES_TABLE_META { - panic("want message type 2 got " + strconv.Itoa(int(result.Type))) - } - data, _ := json.Marshal(result.Meta) - fmt.Println(string(data)) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) - break - } - _, err = db.Exec("insert into example_tmq.t1 values(now,1)") - if err != nil { - panic(err) - } - for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) - } - if result.Type != common.TMQ_RES_DATA { - panic("want message type 1 got " + strconv.Itoa(int(result.Type))) - } - data, _ := json.Marshal(result.Data) - fmt.Println(string(data)) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) - break - } - consumer.Close() } diff --git a/docs/examples/java/pom.xml b/docs/examples/java/pom.xml index 634c3f75a8386db4caab5c1d598f89dc93926c54..01d0ce936c20df1f8567869c1307606ab949cd32 100644 --- a/docs/examples/java/pom.xml +++ b/docs/examples/java/pom.xml @@ -1,6 +1,7 @@ - 4.0.0 @@ -17,13 +18,13 @@ - + com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.1.0 - + junit junit @@ -32,4 +33,4 @@ - + \ No newline at end of file diff --git a/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java new file mode 100644 index 0000000000000000000000000000000000000000..d953a7364160a686d67b4390f3370999b02ce5d4 --- /dev/null +++ b/docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java @@ -0,0 +1,79 @@ +package com.taos.example; + +import com.taosdata.jdbc.tmq.ConsumerRecords; +import com.taosdata.jdbc.tmq.TMQConstants; +import com.taosdata.jdbc.tmq.TaosConsumer; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.Collections; +import java.util.Properties; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicBoolean; + +public class WebsocketSubscribeDemo { + private static final String TOPIC = "tmq_topic_ws"; + private static final String DB_NAME = "meters_ws"; + private static final AtomicBoolean shutdown = new AtomicBoolean(false); + + public static void main(String[] args) { + Timer timer = new Timer(); + timer.schedule(new TimerTask() { + public void run() { + shutdown.set(true); + } + }, 3_000); + try { + // prepare + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + String jdbcUrl = "jdbc:TAOS-RS://127.0.0.1:6041/?user=root&password=taosdata&batchfetch=true"; + try (Connection connection = DriverManager.getConnection(jdbcUrl); + Statement statement = connection.createStatement()) { + statement.executeUpdate("drop topic if exists " + TOPIC); + statement.executeUpdate("drop database if exists " + DB_NAME); + statement.executeUpdate("create database " + DB_NAME); + statement.executeUpdate("use " + DB_NAME); + statement.executeUpdate( + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); + statement.executeUpdate("CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')"); + statement.executeUpdate("INSERT INTO `d0` values(now - 10s, 0.32, 116)"); + statement.executeUpdate("INSERT INTO `d0` values(now - 8s, NULL, NULL)"); + statement.executeUpdate( + "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119)"); + statement.executeUpdate( + "INSERT INTO `d1` values (now-8s, 10, 120) (now - 6s, 10, 119) (now - 4s, 11.2, 118)"); + // create topic + statement.executeUpdate("create topic " + TOPIC + " as select * from meters"); + } + + // create consumer + Properties properties = new Properties(); + properties.setProperty(TMQConstants.BOOTSTRAP_SERVERS, "127.0.0.1:6041"); + properties.setProperty(TMQConstants.CONNECT_TYPE, "ws"); + properties.setProperty(TMQConstants.MSG_WITH_TABLE_NAME, "true"); + properties.setProperty(TMQConstants.ENABLE_AUTO_COMMIT, "true"); + properties.setProperty(TMQConstants.GROUP_ID, "test"); + properties.setProperty(TMQConstants.VALUE_DESERIALIZER, + "com.taos.example.MetersDeserializer"); + + // poll data + try (TaosConsumer consumer = new TaosConsumer<>(properties)) { + consumer.subscribe(Collections.singletonList(TOPIC)); + while (!shutdown.get()) { + ConsumerRecords meters = consumer.poll(Duration.ofMillis(100)); + for (Meters meter : meters) { + System.out.println(meter); + } + } + consumer.unsubscribe(); + } + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + timer.cancel(); + } +} diff --git a/docs/examples/java/src/test/java/com/taos/test/TestAll.java b/docs/examples/java/src/test/java/com/taos/test/TestAll.java index 8d201da0745e1d2d36220c9d78383fc37d4a813a..f24156d8b13fc8f0dfb6eac52b8b68815e8e3483 100644 --- a/docs/examples/java/src/test/java/com/taos/test/TestAll.java +++ b/docs/examples/java/src/test/java/com/taos/test/TestAll.java @@ -64,21 +64,15 @@ public class TestAll { @Test public void testSubscribe() { - - Thread thread = new Thread(() -> { - try { - Thread.sleep(1000); - insertData(); - } catch (SQLException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - }); - thread.start(); SubscribeDemo.main(args); } + + @Test + public void testSubscribeOverWebsocket() { + WebsocketSubscribeDemo.main(args); + } + @Test public void testSchemaless() throws SQLException { LineProtocolExample.main(args); diff --git a/docs/examples/node/restexample/connect.js b/docs/examples/node/restexample/connect.js index bb027d4fe85f35e5a7047b1517caee00314e2153..f36472b4c09a952cffe4b8ab24d1d9aa9f15b3fe 100644 --- a/docs/examples/node/restexample/connect.js +++ b/docs/examples/node/restexample/connect.js @@ -3,18 +3,14 @@ const { options, connect } = require("@tdengine/rest"); async function test() { options.path = "/rest/sql"; options.host = "localhost"; + options.port = 6041; let conn = connect(options); let cursor = conn.cursor(); try { let res = await cursor.query("SELECT server_version()"); - res.toString(); + console.log("res.getResult()",res.getResult()); } catch (err) { console.log(err); } } test(); - -// output: -// server_version() | -// =================== -// 3.0.0.0 | diff --git a/docs/examples/node/restexample/insert_example.js b/docs/examples/node/restexample/insert_example.js new file mode 100644 index 0000000000000000000000000000000000000000..d9fddf9f2888a878f369e7a2ee20c5ce49d0cc99 --- /dev/null +++ b/docs/examples/node/restexample/insert_example.js @@ -0,0 +1,19 @@ +const { options, connect } = require("@tdengine/rest"); + +async function sqlInsert() { + options.path = "/rest/sql"; + options.host = "localhost"; + options.port = 6041; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query('CREATE DATABASE power'); + res = await cursor.query('CREATE STABLE power.meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupId int)'); + res = await cursor.query('INSERT INTO power.d1001 USING power.meters TAGS ("California.SanFrancisco", 2) VALUES (NOW, 10.2, 219, 0.32)'); + console.log("res.getResult()", res.getResult()); + } catch (err) { + console.log(err); + } +} +sqlInsert(); + diff --git a/docs/examples/node/restexample/query_example.js b/docs/examples/node/restexample/query_example.js new file mode 100644 index 0000000000000000000000000000000000000000..0edce64a24edddaf48b207d676f6d704d832ed68 --- /dev/null +++ b/docs/examples/node/restexample/query_example.js @@ -0,0 +1,16 @@ +const { options, connect } = require("@tdengine/rest"); + +async function query() { + options.path = "/rest/sql"; + options.host = "localhost"; + options.port = 6041; + let conn = connect(options); + let cursor = conn.cursor(); + try { + let res = await cursor.query('select * from power.meters'); + console.log("res.getResult()", res.getResult()); + } catch (err) { + console.log(err); + } +} +query(); diff --git a/docs/examples/node/restexample/restChecker.js b/docs/examples/node/restexample/restChecker.js new file mode 100644 index 0000000000000000000000000000000000000000..a999684e575d4271e5a62895a5b1678a37d16a65 --- /dev/null +++ b/docs/examples/node/restexample/restChecker.js @@ -0,0 +1,78 @@ +const { options, connect } = require("@tdengine/rest"); +options.path = '/rest/sql/' +options.host = 'localhost'; +options.port = 6041; +options.user = "root"; +options.passwd = "taosdata"; + +//optional +// options.url = "http://127.0.0.1:6041"; + +const db = 'rest_ts_db'; +const table = 'rest' +const createDB = `create database if not exists ${db} keep 3650`; +const dropDB = `drop database if exists ${db}`; +const createTB = `create table if not exists ${db}.${table}(ts timestamp,i8 tinyint,i16 smallint,i32 int,i64 bigint,bnr binary(40),nchr nchar(40))`; +const addColumn = `alter table ${db}.${table} add column new_column nchar(40) `; +const dropColumn = `alter table ${db}.${table} drop column new_column`; +const insertSql = `insert into ${db}.${table} values('2022-03-30 18:30:51.567',1,2,3,4,'binary1','nchar1')` + + `('2022-03-30 18:30:51.568',5,6,7,8,'binary2','nchar2')` + + `('2022-03-30 18:30:51.569',9,0,1,2,'binary3','nchar3')`; +const querySql = `select * from ${db}.${table}`; +const errorSql = 'show database'; + +let conn = connect(options); +let cursor = conn.cursor(); + +async function execute(sql, pure = true) { + let result = await cursor.query(sql, pure); + // print query result as taos shell + // Get Result object, return Result object. + console.log("result.getResult()",result.getResult()); + // Get Meta data, return Meta[]|undefined(when execute failed this is undefined). + console.log("result.getMeta()",result.getMeta()); + // Get data,return Array>|undefined(when execute failed this is undefined). + console.log("result.getData()",result.getData()); + // Get affect rows,return number|undefined(when execute failed this is undefined). + console.log("result.getAffectRows()",result.getAffectRows()); + // Get command,return SQL send to server(need to `query(sql,false)`,set 'pure=false',default true). + console.log("result.getCommand()",result.getCommand()); + // Get error code ,return number|undefined(when execute failed this is undefined). + console.log("result.getErrCode()",result.getErrCode()); + // Get error string,return string|undefined(when execute failed this is undefined). + console.log("result.getErrStr()",result.getErrStr()); +} + +(async () => { + // start execute time + let start = new Date().getTime(); + await execute(createDB); + console.log("-----------------------------------") + + await execute(createTB); + console.log("-----------------------------------") + + await execute(addColumn); + console.log("----------------------------------") + + await execute(dropColumn); + console.log("-----------------------------------") + + await execute(insertSql); + console.log("-----------------------------------") + + await execute(querySql); + console.log("-----------------------------------") + + await execute(errorSql); + console.log("-----------------------------------") + + await execute(dropDB); + // finish time + let end = new Date().getTime(); + console.log("total spend time:%d ms",end - start); +})() + + + + diff --git a/docs/examples/python/kafka_example.py b/docs/examples/python/kafka_example.py deleted file mode 100644 index a89287d372d505225e2d679e7f205a3857014533..0000000000000000000000000000000000000000 --- a/docs/examples/python/kafka_example.py +++ /dev/null @@ -1,242 +0,0 @@ -#! encoding = utf-8 -import json -import time -from json import JSONDecodeError -from typing import Callable -import logging -from concurrent.futures import ThreadPoolExecutor, Future - -import taos -from kafka import KafkaConsumer -from kafka.consumer.fetcher import ConsumerRecord - - -class Consumer(object): - DEFAULT_CONFIGS = { - 'kafka_brokers': 'localhost:9092', - 'kafka_topic': 'python_kafka', - 'kafka_group_id': 'taos', - 'taos_host': 'localhost', - 'taos_user': 'root', - 'taos_password': 'taosdata', - 'taos_database': 'power', - 'taos_port': 6030, - 'timezone': None, - 'clean_after_testing': False, - 'bath_consume': True, - 'batch_size': 1000, - 'async_model': True, - 'workers': 10, - 'testing': False - } - - LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', - 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', - 'California.SantaClara', 'California.Cupertino'] - - CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1' - USE_DATABASE_SQL = 'use {}' - DROP_TABLE_SQL = 'drop table if exists meters' - DROP_DATABASE_SQL = 'drop database if exists {}' - CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) ' \ - 'tags (location binary(64), groupId int)' - CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})' - INSERT_SQL_HEADER = "insert into " - INSERT_PART_SQL = 'power.{} values (\'{}\', {}, {}, {})' - - def __init__(self, **configs): - self.config: dict = self.DEFAULT_CONFIGS - self.config.update(configs) - if not self.config.get('testing'): - self.consumer = KafkaConsumer( - self.config.get('kafka_topic'), # topic - bootstrap_servers=self.config.get('kafka_brokers'), - group_id=self.config.get('kafka_group_id'), - ) - self.taos = taos.connect( - host=self.config.get('taos_host'), - user=self.config.get('taos_user'), - password=self.config.get('taos_password'), - port=self.config.get('taos_port'), - timezone=self.config.get('timezone'), - ) - if self.config.get('async_model'): - self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers')) - self.tasks = [] - # tags and table mapping # key: {location}_{groupId} value: - self.tag_table_mapping = {} - i = 0 - for location in self.LOCATIONS: - for j in range(1, 11): - table_name = 'd{}'.format(i) - self._cache_table(location=location, group_id=j, table_name=table_name) - i += 1 - - def init_env(self): - # create database and table - self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database'))) - self.taos.execute(self.CREATE_DATABASE_SQL.format(self.config.get('taos_database'))) - self.taos.execute(self.USE_DATABASE_SQL.format(self.config.get('taos_database'))) - self.taos.execute(self.DROP_TABLE_SQL) - self.taos.execute(self.CREATE_STABLE_SQL) - for tags, table_name in self.tag_table_mapping.items(): - location, group_id = _get_location_and_group(tags) - self.taos.execute(self.CREATE_TABLE_SQL.format(table_name, location, group_id)) - - def consume(self): - logging.warning('## start consumer topic-[%s]', self.config.get('kafka_topic')) - try: - if self.config.get('bath_consume'): - self._run_batch(self._to_taos_batch) - else: - self._run(self._to_taos) - except KeyboardInterrupt: - logging.warning("## caught keyboard interrupt, stopping") - finally: - self.stop() - - def stop(self): - # close consumer - if self.consumer is not None: - self.consumer.commit() - self.consumer.close() - - # multi thread - if self.config.get('async_model'): - for task in self.tasks: - while not task.done(): - pass - if self.pool is not None: - self.pool.shutdown() - - # clean data - if self.config.get('clean_after_testing'): - self.taos.execute(self.DROP_TABLE_SQL) - self.taos.execute(self.DROP_DATABASE_SQL.format(self.config.get('taos_database'))) - # close taos - if self.taos is not None: - self.taos.close() - - def _run(self, f): - for message in self.consumer: - if self.config.get('async_model'): - self.pool.submit(f(message)) - else: - f(message) - - def _run_batch(self, f): - while True: - messages = self.consumer.poll(timeout_ms=500, max_records=self.config.get('batch_size')) - if messages: - if self.config.get('async_model'): - self.pool.submit(f, messages.values()) - else: - f(list(messages.values())) - if not messages: - time.sleep(0.1) - - def _to_taos(self, message: ConsumerRecord) -> bool: - sql = self.INSERT_SQL_HEADER + self._build_sql(message.value) - if len(sql) == 0: # decode error, skip - return True - logging.info('## insert sql %s', sql) - return self.taos.execute(sql=sql) == 1 - - def _to_taos_batch(self, messages): - sql = self._build_sql_batch(messages=messages) - if len(sql) == 0: # decode error, skip - return - self.taos.execute(sql=sql) - - def _build_sql(self, msg_value: str) -> str: - try: - data = json.loads(msg_value) - except JSONDecodeError as e: - logging.error('## decode message [%s] error ', msg_value, e) - return '' - location = data.get('location') - group_id = data.get('groupId') - ts = data.get('ts') - current = data.get('current') - voltage = data.get('voltage') - phase = data.get('phase') - - table_name = self._get_table_name(location=location, group_id=group_id) - return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase) - - def _build_sql_batch(self, messages) -> str: - sql_list = [] - for partition_messages in messages: - for message in partition_messages: - sql_list.append(self._build_sql(message.value)) - - return self.INSERT_SQL_HEADER + ' '.join(sql_list) - - def _cache_table(self, location: str, group_id: int, table_name: str): - self.tag_table_mapping[_tag_table_mapping_key(location=location, group_id=group_id)] = table_name - - def _get_table_name(self, location: str, group_id: int) -> str: - return self.tag_table_mapping.get(_tag_table_mapping_key(location=location, group_id=group_id)) - - -def _tag_table_mapping_key(location: str, group_id: int): - return '{}_{}'.format(location, group_id) - - -def _get_location_and_group(key: str) -> (str, int): - fields = key.split('_') - return fields[0], fields[1] - - -def test_to_taos(consumer: Consumer): - msg = { - 'location': 'California.SanFrancisco', - 'groupId': 1, - 'ts': '2022-12-06 15:13:38.643', - 'current': 3.41, - 'voltage': 105, - 'phase': 0.02027, - } - record = ConsumerRecord(checksum=None, headers=None, offset=1, key=None, value=json.dumps(msg), partition=1, - topic='test', serialized_key_size=None, serialized_header_size=None, - serialized_value_size=None, timestamp=time.time(), timestamp_type=None) - assert consumer._to_taos(message=record) - - -def test_to_taos_batch(consumer: Consumer): - records = [ - [ - ConsumerRecord(checksum=None, headers=None, offset=1, key=None, - value=json.dumps({'location': 'California.SanFrancisco', - 'groupId': 1, - 'ts': '2022-12-06 15:13:38.643', - 'current': 3.41, - 'voltage': 105, - 'phase': 0.02027, }), - partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, - serialized_value_size=None, timestamp=time.time(), timestamp_type=None), - ConsumerRecord(checksum=None, headers=None, offset=1, key=None, - value=json.dumps({'location': 'California.LosAngles', - 'groupId': 2, - 'ts': '2022-12-06 15:13:39.643', - 'current': 3.41, - 'voltage': 102, - 'phase': 0.02027, }), - partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, - serialized_value_size=None, timestamp=time.time(), timestamp_type=None), - ] - ] - - consumer._to_taos_batch(messages=records) - - -if __name__ == '__main__': - consumer = Consumer(async_model=True, testing=True) - # init env - consumer.init_env() - # consumer.consume() - # test build sql - # test build sql batch - test_to_taos(consumer) - test_to_taos_batch(consumer) - \ No newline at end of file diff --git a/docs/examples/python/kafka_example_common.py b/docs/examples/python/kafka_example_common.py new file mode 100644 index 0000000000000000000000000000000000000000..566748c94e2542aabe8265ed55c85e4b725d69bb --- /dev/null +++ b/docs/examples/python/kafka_example_common.py @@ -0,0 +1,65 @@ +#! encoding = utf-8 +import taos + +LOCATIONS = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', + 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', + 'California.SantaClara', 'California.Cupertino'] + +CREATE_DATABASE_SQL = 'create database if not exists {} keep 365 duration 10 buffer 16 wal_level 1' +USE_DATABASE_SQL = 'use {}' +DROP_TABLE_SQL = 'drop table if exists meters' +DROP_DATABASE_SQL = 'drop database if exists {}' +CREATE_STABLE_SQL = 'create stable meters (ts timestamp, current float, voltage int, phase float) tags ' \ + '(location binary(64), groupId int)' +CREATE_TABLE_SQL = 'create table if not exists {} using meters tags (\'{}\', {})' + + +def create_database_and_tables(host, port, user, password, db, table_count): + tags_tables = _init_tags_table_names(table_count=table_count) + conn = taos.connect(host=host, port=port, user=user, password=password) + + conn.execute(DROP_DATABASE_SQL.format(db)) + conn.execute(CREATE_DATABASE_SQL.format(db)) + conn.execute(USE_DATABASE_SQL.format(db)) + conn.execute(DROP_TABLE_SQL) + conn.execute(CREATE_STABLE_SQL) + for tags in tags_tables: + location, group_id = _get_location_and_group(tags) + tables = tags_tables[tags] + for table_name in tables: + conn.execute(CREATE_TABLE_SQL.format(table_name, location, group_id)) + conn.close() + + +def clean(host, port, user, password, db): + conn = taos.connect(host=host, port=port, user=user, password=password) + conn.execute(DROP_DATABASE_SQL.format(db)) + conn.close() + + +def _init_tags_table_names(table_count): + tags_table_names = {} + group_id = 0 + for i in range(table_count): + table_name = 'd{}'.format(i) + location_idx = i % len(LOCATIONS) + location = LOCATIONS[location_idx] + if location_idx == 0: + group_id += 1 + if group_id > 10: + group_id -= 10 + key = _tag_table_mapping_key(location=location, group_id=group_id) + if key not in tags_table_names: + tags_table_names[key] = [] + tags_table_names[key].append(table_name) + + return tags_table_names + + +def _tag_table_mapping_key(location, group_id): + return '{}_{}'.format(location, group_id) + + +def _get_location_and_group(key): + fields = key.split('_') + return fields[0], fields[1] diff --git a/docs/examples/python/kafka_example_consumer.py b/docs/examples/python/kafka_example_consumer.py new file mode 100644 index 0000000000000000000000000000000000000000..e2d5cf535b3953a3c0ecec9e25cc615948162633 --- /dev/null +++ b/docs/examples/python/kafka_example_consumer.py @@ -0,0 +1,231 @@ +#! encoding = utf-8 +import json +import logging +import time +from concurrent.futures import ThreadPoolExecutor, Future +from json import JSONDecodeError +from typing import Callable + +import taos +from kafka import KafkaConsumer +from kafka.consumer.fetcher import ConsumerRecord + +import kafka_example_common as common + + +class Consumer(object): + DEFAULT_CONFIGS = { + 'kafka_brokers': 'localhost:9092', # kafka broker + 'kafka_topic': 'tdengine_kafka_practices', + 'kafka_group_id': 'taos', + 'taos_host': 'localhost', # TDengine host + 'taos_port': 6030, # TDengine port + 'taos_user': 'root', # TDengine user name + 'taos_password': 'taosdata', # TDengine password + 'taos_database': 'power', # TDengine database + 'message_type': 'json', # message format, 'json' or 'line' + 'clean_after_testing': False, # if drop database after testing + 'max_poll': 1000, # poll size for batch mode + 'workers': 10, # thread count for multi-threading + 'testing': False + } + + INSERT_SQL_HEADER = "insert into " + INSERT_PART_SQL = '{} values (\'{}\', {}, {}, {})' + + def __init__(self, **configs): + self.config = self.DEFAULT_CONFIGS + self.config.update(configs) + + self.consumer = None + if not self.config.get('testing'): + self.consumer = KafkaConsumer( + self.config.get('kafka_topic'), + bootstrap_servers=self.config.get('kafka_brokers'), + group_id=self.config.get('kafka_group_id'), + ) + + self.conns = taos.connect( + host=self.config.get('taos_host'), + port=self.config.get('taos_port'), + user=self.config.get('taos_user'), + password=self.config.get('taos_password'), + db=self.config.get('taos_database'), + ) + if self.config.get('workers') > 1: + self.pool = ThreadPoolExecutor(max_workers=self.config.get('workers')) + self.tasks = [] + # tags and table mapping # key: {location}_{groupId} value: + + def consume(self): + """ + + consume data from kafka and deal. Base on `message_type`, `bath_consume`, `insert_by_table`, + there are several deal function. + :return: + """ + self.conns.execute(common.USE_DATABASE_SQL.format(self.config.get('taos_database'))) + try: + if self.config.get('message_type') == 'line': # line + self._run(self._line_to_taos) + if self.config.get('message_type') == 'json': # json + self._run(self._json_to_taos) + except KeyboardInterrupt: + logging.warning("## caught keyboard interrupt, stopping") + finally: + self.stop() + + def stop(self): + """ + + stop consuming + :return: + """ + # close consumer + if self.consumer is not None: + self.consumer.commit() + self.consumer.close() + + # multi thread + if self.config.get('workers') > 1: + if self.pool is not None: + self.pool.shutdown() + for task in self.tasks: + while not task.done(): + time.sleep(0.01) + + # clean data + if self.config.get('clean_after_testing'): + self.conns.execute(common.DROP_TABLE_SQL) + self.conns.execute(common.DROP_DATABASE_SQL.format(self.config.get('taos_database'))) + # close taos + if self.conns is not None: + self.conns.close() + + def _run(self, f): + """ + + run in batch consuming mode + :param f: + :return: + """ + i = 0 # just for test. + while True: + messages = self.consumer.poll(timeout_ms=100, max_records=self.config.get('max_poll')) + if messages: + if self.config.get('workers') > 1: + self.pool.submit(f, messages.values()) + else: + f(list(messages.values())) + if not messages: + i += 1 # just for test. + time.sleep(0.1) + if i > 3: # just for test. + logging.warning('## test over.') # just for test. + return # just for test. + + def _json_to_taos(self, messages): + """ + + convert a batch of json data to sql, and insert into TDengine + :param messages: + :return: + """ + sql = self._build_sql_from_json(messages=messages) + self.conns.execute(sql=sql) + + def _line_to_taos(self, messages): + """ + + convert a batch of lines data to sql, and insert into TDengine + :param messages: + :return: + """ + lines = [] + for partition_messages in messages: + for message in partition_messages: + lines.append(message.value.decode()) + sql = self.INSERT_SQL_HEADER + ' '.join(lines) + self.conns.execute(sql=sql) + + def _build_single_sql_from_json(self, msg_value): + try: + data = json.loads(msg_value) + except JSONDecodeError as e: + logging.error('## decode message [%s] error ', msg_value, e) + return '' + # location = data.get('location') + # group_id = data.get('groupId') + ts = data.get('ts') + current = data.get('current') + voltage = data.get('voltage') + phase = data.get('phase') + table_name = data.get('table_name') + + return self.INSERT_PART_SQL.format(table_name, ts, current, voltage, phase) + + def _build_sql_from_json(self, messages): + sql_list = [] + for partition_messages in messages: + for message in partition_messages: + sql_list.append(self._build_single_sql_from_json(message.value)) + return self.INSERT_SQL_HEADER + ' '.join(sql_list) + + +def test_json_to_taos(consumer: Consumer): + records = [ + [ + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value=json.dumps({'table_name': 'd0', + 'ts': '2022-12-06 15:13:38.643', + 'current': 3.41, + 'voltage': 105, + 'phase': 0.02027, }), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value=json.dumps({'table_name': 'd1', + 'ts': '2022-12-06 15:13:39.643', + 'current': 3.41, + 'voltage': 102, + 'phase': 0.02027, }), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ] + ] + + consumer._json_to_taos(messages=records) + + +def test_line_to_taos(consumer: Consumer): + records = [ + [ + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value="d0 values('2023-01-01 00:00:00.001', 3.49, 109, 0.02737)".encode('utf-8'), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ConsumerRecord(checksum=None, headers=None, offset=1, key=None, + value="d1 values('2023-01-01 00:00:00.002', 6.19, 112, 0.09171)".encode('utf-8'), + partition=1, topic='test', serialized_key_size=None, serialized_header_size=None, + serialized_value_size=None, timestamp=time.time(), timestamp_type=None), + ] + ] + consumer._line_to_taos(messages=records) + + +def consume(kafka_brokers, kafka_topic, kafka_group_id, taos_host, taos_port, taos_user, + taos_password, taos_database, message_type, max_poll, workers): + c = Consumer(kafka_brokers=kafka_brokers, kafka_topic=kafka_topic, kafka_group_id=kafka_group_id, + taos_host=taos_host, taos_port=taos_port, taos_user=taos_user, taos_password=taos_password, + taos_database=taos_database, message_type=message_type, max_poll=max_poll, workers=workers) + c.consume() + + +if __name__ == '__main__': + consumer = Consumer(testing=True) + common.create_database_and_tables(host='localhost', port=6030, user='root', password='taosdata', db='py_kafka_test', + table_count=10) + consumer.conns.execute(common.USE_DATABASE_SQL.format('py_kafka_test')) + test_json_to_taos(consumer) + test_line_to_taos(consumer) + common.clean(host='localhost', port=6030, user='root', password='taosdata', db='py_kafka_test') diff --git a/docs/examples/python/kafka_example_perform.py b/docs/examples/python/kafka_example_perform.py new file mode 100644 index 0000000000000000000000000000000000000000..23ae4b48c8fc8139b85cd41b041953e8f55f12b4 --- /dev/null +++ b/docs/examples/python/kafka_example_perform.py @@ -0,0 +1,103 @@ +#! encoding=utf-8 + +import argparse +import logging +import multiprocessing +import time +from multiprocessing import pool + +import kafka_example_common as common +import kafka_example_consumer as consumer +import kafka_example_producer as producer + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-kafka-broker', type=str, default='localhost:9092', + help='kafka borker host. default is `localhost:9200`') + parser.add_argument('-kafka-topic', type=str, default='tdengine-kafka-practices', + help='kafka topic. default is `tdengine-kafka-practices`') + parser.add_argument('-kafka-group', type=str, default='kafka_practices', + help='kafka consumer group. default is `kafka_practices`') + parser.add_argument('-taos-host', type=str, default='localhost', + help='TDengine host. default is `localhost`') + parser.add_argument('-taos-port', type=int, default=6030, help='TDengine port. default is 6030') + parser.add_argument('-taos-user', type=str, default='root', help='TDengine username, default is `root`') + parser.add_argument('-taos-password', type=str, default='taosdata', help='TDengine password, default is `taosdata`') + parser.add_argument('-taos-db', type=str, default='tdengine_kafka_practices', + help='TDengine db name, default is `tdengine_kafka_practices`') + parser.add_argument('-table-count', type=int, default=100, help='TDengine sub-table count, default is 100') + parser.add_argument('-table-items', type=int, default=1000, help='items in per sub-tables, default is 1000') + parser.add_argument('-message-type', type=str, default='line', + help='kafka message type. `line` or `json`. default is `line`') + parser.add_argument('-max-poll', type=int, default=1000, help='max poll for kafka consumer') + parser.add_argument('-threads', type=int, default=10, help='thread count for deal message') + parser.add_argument('-processes', type=int, default=1, help='process count') + + args = parser.parse_args() + total = args.table_count * args.table_items + + logging.warning("## start to prepare testing data...") + prepare_data_start = time.time() + producer.produce_total(100, args.kafka_broker, args.kafka_topic, args.message_type, total, args.table_count) + prepare_data_end = time.time() + logging.warning("## prepare testing data finished! spend-[%s]", prepare_data_end - prepare_data_start) + + logging.warning("## start to create database and tables ...") + create_db_start = time.time() + # create database and table + common.create_database_and_tables(host=args.taos_host, port=args.taos_port, user=args.taos_user, + password=args.taos_password, db=args.taos_db, table_count=args.table_count) + create_db_end = time.time() + logging.warning("## create database and tables finished! spend [%s]", create_db_end - create_db_start) + + processes = args.processes + + logging.warning("## start to consume data and insert into TDengine...") + consume_start = time.time() + if processes > 1: # multiprocess + multiprocessing.set_start_method("spawn") + pool = pool.Pool(processes) + + consume_start = time.time() + for _ in range(processes): + pool.apply_async(func=consumer.consume, args=( + args.kafka_broker, args.kafka_topic, args.kafka_group, args.taos_host, args.taos_port, args.taos_user, + args.taos_password, args.taos_db, args.message_type, args.max_poll, args.threads)) + pool.close() + pool.join() + else: + consume_start = time.time() + consumer.consume(kafka_brokers=args.kafka_broker, kafka_topic=args.kafka_topic, kafka_group_id=args.kafka_group, + taos_host=args.taos_host, taos_port=args.taos_port, taos_user=args.taos_user, + taos_password=args.taos_password, taos_database=args.taos_db, message_type=args.message_type, + max_poll=args.max_poll, workers=args.threads) + consume_end = time.time() + logging.warning("## consume data and insert into TDengine over! spend-[%s]", consume_end - consume_start) + + # print report + logging.warning( + "\n#######################\n" + " Prepare data \n" + "#######################\n" + "# data_type # %s \n" + "# total # %s \n" + "# spend # %s s\n" + "#######################\n" + " Create database \n" + "#######################\n" + "# stable # 1 \n" + "# sub-table # 100 \n" + "# spend # %s s \n" + "#######################\n" + " Consume \n" + "#######################\n" + "# data_type # %s \n" + "# threads # %s \n" + "# processes # %s \n" + "# total_count # %s \n" + "# spend # %s s\n" + "# per_second # %s \n" + "#######################\n", + args.message_type, total, prepare_data_end - prepare_data_start, create_db_end - create_db_start, + args.message_type, args.threads, processes, total, consume_end - consume_start, + total / (consume_end - consume_start)) diff --git a/docs/examples/python/kafka_example_producer.py b/docs/examples/python/kafka_example_producer.py new file mode 100644 index 0000000000000000000000000000000000000000..51468c7e37ab3400bece69fa58e126a789ef9860 --- /dev/null +++ b/docs/examples/python/kafka_example_producer.py @@ -0,0 +1,97 @@ +#! encoding = utf-8 +import json +import random +import threading +from concurrent.futures import ThreadPoolExecutor, Future +from datetime import datetime + +from kafka import KafkaProducer + +locations = ['California.SanFrancisco', 'California.LosAngles', 'California.SanDiego', 'California.SanJose', + 'California.PaloAlto', 'California.Campbell', 'California.MountainView', 'California.Sunnyvale', + 'California.SantaClara', 'California.Cupertino'] + +producers: list[KafkaProducer] = [] + +lock = threading.Lock() +start = 1640966400 + + +def produce_total(workers, broker, topic, message_type, total, table_count): + if len(producers) == 0: + lock.acquire() + if len(producers) == 0: + _init_kafka_producers(broker=broker, count=10) + lock.release() + pool = ThreadPoolExecutor(max_workers=workers) + futures = [] + for _ in range(0, workers): + futures.append(pool.submit(_produce_total, topic, message_type, int(total / workers), table_count)) + pool.shutdown() + for f in futures: + f.result() + _close_kafka_producers() + + +def _produce_total(topic, message_type, total, table_count): + producer = _get_kafka_producer() + for _ in range(total): + message = _get_fake_date(message_type=message_type, table_count=table_count) + producer.send(topic=topic, value=message.encode(encoding='utf-8')) + + +def _init_kafka_producers(broker, count): + for _ in range(count): + p = KafkaProducer(bootstrap_servers=broker, batch_size=64 * 1024, linger_ms=300, acks=0) + producers.append(p) + + +def _close_kafka_producers(): + for p in producers: + p.close() + + +def _get_kafka_producer(): + return producers[random.randint(0, len(producers) - 1)] + + +def _get_fake_date(table_count, message_type='json'): + if message_type == 'json': + return _get_json_message(table_count=table_count) + if message_type == 'line': + return _get_line_message(table_count=table_count) + return '' + + +def _get_json_message(table_count): + return json.dumps({ + 'ts': _get_timestamp(), + 'current': random.randint(0, 1000) / 100, + 'voltage': random.randint(105, 115), + 'phase': random.randint(0, 32000) / 100000, + 'location': random.choice(locations), + 'groupId': random.randint(1, 10), + 'table_name': _random_table_name(table_count) + }) + + +def _get_line_message(table_count): + return "{} values('{}', {}, {}, {})".format( + _random_table_name(table_count), # table + _get_timestamp(), # ts + random.randint(0, 1000) / 100, # current + random.randint(105, 115), # voltage + random.randint(0, 32000) / 100000, # phase + ) + + +def _random_table_name(table_count): + return 'd{}'.format(random.randint(0, table_count - 1)) + + +def _get_timestamp(): + global start + lock.acquire(blocking=True) + start += 0.001 + lock.release() + return datetime.fromtimestamp(start).strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] diff --git a/docs/examples/python/mockdatasource.py b/docs/examples/python/mockdatasource.py index 15a7d2ff8c5b1febefcf4f81f481220947257b50..9c702936ea6f1bdff3f604d376fd1925b4dc118e 100644 --- a/docs/examples/python/mockdatasource.py +++ b/docs/examples/python/mockdatasource.py @@ -59,4 +59,3 @@ if __name__ == '__main__': datasource = MockDataSource('t', 10, False) for data in datasource: print(data) - \ No newline at end of file diff --git a/docs/examples/python/native_insert_example.py b/docs/examples/python/native_insert_example.py index 94fd00a6e9d1dcd2119693c4b5c862d36c219a3d..cdde7d23d24d12e11c67b6c6acc0e0b089fb5335 100644 --- a/docs/examples/python/native_insert_example.py +++ b/docs/examples/python/native_insert_example.py @@ -25,10 +25,10 @@ def create_stable(conn: taos.TaosConnection): # The generated SQL is: -# INSERT INTO d1001 USING meters TAGS(California.SanFrancisco, 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) -# d1002 USING meters TAGS(California.SanFrancisco, 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) -# d1003 USING meters TAGS(California.LosAngeles, 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) -# d1004 USING meters TAGS(California.LosAngeles, 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) +# INSERT INTO d1001 USING meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) +# d1002 USING meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) +# d1003 USING meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) +# d1004 USING meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000) def get_sql(): global lines diff --git a/docs/examples/python/sql_writer.py b/docs/examples/python/sql_writer.py index db51bb71744c89ed130370f611f655086de0ef74..3456981a7b9a174e38f8795ff7251ab3c675174b 100644 --- a/docs/examples/python/sql_writer.py +++ b/docs/examples/python/sql_writer.py @@ -109,4 +109,3 @@ if __name__ == '__main__': writer.execute_sql( "INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) " "VALUES ('2021-07-13 14:06:32.272', 10.2, 219, 0.32)") - \ No newline at end of file diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 32778e9f25f4a8b58007f39d4aeac53c30fe6895..6f7fb87c89ce4cb96793d09a837f60ad54ae69bc 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -52,4 +52,4 @@ if __name__ == '__main__': finally: consumer.unsubscribe() consumer.close() - cleanup("tmq_test", "tmq_test_topic") \ No newline at end of file + cleanup("tmq_test", "tmq_test_topic") diff --git a/docs/examples/python/tmq_websocket_example.py b/docs/examples/python/tmq_websocket_example.py new file mode 100644 index 0000000000000000000000000000000000000000..e1dcb0086a995c0c20a5d079ed6d8f4d18ea0356 --- /dev/null +++ b/docs/examples/python/tmq_websocket_example.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 +from taosws import Consumer + +conf = { + "td.connect.websocket.scheme": "ws", + "group.id": "0", +} +consumer = Consumer(conf) + +consumer.subscribe(["test"]) + +while True: + message = consumer.poll(timeout=1.0) + if message: + id = message.vgroup() + topic = message.topic() + database = message.database() + + for block in message: + nrows = block.nrows() + ncols = block.ncols() + for row in block: + print(row) + values = block.fetchall() + print(nrows, ncols) + + # consumer.commit(message) + else: + break + +consumer.close() diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 781a65402c24d368b8f62bda825875f3cd5a6aac..7edb9b62f468f7388a2343f38d919c307a3edb35 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -10,7 +10,7 @@ import PkgListV3 from "/components/PkgListV3"; 您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣,请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. -TDengine 完整的软件包包括服务端(taosd)、应用驱动(taosc)、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序(CLI,taos)和一些工具软件。目前 taosdump、TDinsight 仅在 Linux 系统上安装和运行,后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外,还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 +TDengine 完整的软件包包括服务端(taosd)、应用驱动(taosc)、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序(CLI,taos)和一些工具软件。目前 TDinsight 仅在 Linux 系统上安装和运行,后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外,还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 为方便使用,标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码;如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持,也可以仅下载 Lite 版本的安装包。 diff --git a/docs/zh/05-get-started/index.md b/docs/zh/05-get-started/index.md index 832310aa7c677940c7e4ca13be5f31c2d98a64dc..e144c563b97304f6257d3a1989d7caf85d3789aa 100644 --- a/docs/zh/05-get-started/index.md +++ b/docs/zh/05-get-started/index.md @@ -4,6 +4,7 @@ description: '快速设置 TDengine 环境并体验其高效写入和查询' --- import xiaot from './xiaot.webp' +import xiaot_new from './xiaot-new.webp' import channel from './channel.webp' import official_account from './official-account.webp' @@ -35,13 +36,13 @@ TDengine 知识地图中涵盖了 TDengine 的各种知识点,揭示了各概
- + - - - + + +
小 T 的二维码小 T 的二维码 TDengine 微信视频号 TDengine 微信公众号
加入“物联网大数据技术前沿群”
与大家进行技术交流
关注 TDengine 微信视频号
收看技术直播与教学视频
关注 TDengine 微信公众号
阅读核心技术与行业案例文章
加入“物联网大数据技术群”
与大家进行技术交流
关注 TDengine 视频号
收看技术直播与教学视频
关注 TDengine 公众号
阅读技术文章与行业案例
diff --git a/docs/zh/05-get-started/xiaot-new.webp b/docs/zh/05-get-started/xiaot-new.webp new file mode 100644 index 0000000000000000000000000000000000000000..483b54d2ef3d8894527aa154a42cf6cd2463c579 Binary files /dev/null and b/docs/zh/05-get-started/xiaot-new.webp differ diff --git a/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx b/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx index 47024171562bb7dc2d4e2063bc2fb0d44faad45b..89974ceb7b77627c70c4ec468b9716c4f2b204fc 100644 --- a/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx +++ b/docs/zh/07-develop/03-insert-data/01-sql-writing.mdx @@ -30,25 +30,31 @@ import PhpStmt from "./_php_stmt.mdx"; 下面这条 INSERT 就将一条记录写入到表 d1001 中: ```sql -INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31); +INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31); ``` +这里的`ts1`为Unix时间戳(Unix timestamp),允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的 KEEP 值。时间戳详情规则参考 [TDengine SQL数据写入 关于时间戳一节](/taos-sql/insert) + ### 一次写入多条 TDengine 支持一次写入多条记录,比如下面这条命令就将两条记录写入到表 d1001 中: ```sql -INSERT INTO d1001 VALUES (1538548684000, 10.2, 220, 0.23) (1538548696650, 10.3, 218, 0.25); +INSERT INTO d1001 VALUES (ts1, 10.2, 220, 0.23) (ts2, 10.3, 218, 0.25); ``` +这里的`ts1`和`ts2`为Unix时间戳(Unix timestamp),允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的 KEEP 值。时间戳详情规则参考 [TDengine SQL数据写入 关于时间戳一节](/taos-sql/insert) + ### 一次写入多表 TDengine 也支持一次向多个表写入数据,比如下面这条命令就向 d1001 写入两条记录,向 d1002 写入一条记录: ```sql -INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); +INSERT INTO d1001 VALUES (ts1, 10.3, 219, 0.31) (ts2, 12.6, 218, 0.33) d1002 VALUES (ts3, 12.3, 221, 0.31); ``` +这里的`ts1`、`ts2`和`ts3`为Unix时间戳(Unix timestamp),允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的 KEEP 值。时间戳详情规则参考 [TDengine SQL数据写入 关于时间戳一节](/taos-sql/insert) + 详细的 SQL INSERT 语法规则参考 [TDengine SQL 的数据写入](/taos-sql/insert)。 :::info diff --git a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx index afe73af8dbc3a768a3e7640a9de5f11dbe18653c..876f123fe13776b5ccb045fc390182e8bc8ecf8e 100644 --- a/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/30-influxdb-line.mdx @@ -37,7 +37,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - tag_set 中的所有的数据自动转化为 NCHAR 数据类型; - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 FLOAT 类型的数值 1.2, 如果不带类型后缀会被当作 DOUBLE 处理; - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度,支持从小时到纳秒的 6 种时间精度。 -- 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field,后面的数据按照这个顺序),如果顺序不一样,需要配置参数 smlDataFormat 为 false,否则,数据写入按照相同顺序写入,库中数据会异常。(3.0.1.3 之后的版本 smlDataFormat 默认为 false) [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field,后面的数据按照这个顺序),如果顺序不一样,需要配置参数 smlDataFormat 为 false,否则,数据写入按照相同顺序写入,库中数据会异常。(3.0.1.3 之后的版本 smlDataFormat 默认为 false,从3.0.3.0开始,该配置废弃) [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) - 默认产生的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下:假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的子表名为 cpu1。注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: diff --git a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx index 89818409c5032166e77a50f07ea1b54bd66617cb..e1fd3dacc8d91ea4ce3efde6f71a645cc65140ea 100644 --- a/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -47,7 +47,6 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 :::note - 对于 JSON 格式协议,TDengine 并不会自动把所有标签转成 NCHAR 类型, 字符串将将转为 NCHAR 类型, 数值将同样转换为 DOUBLE 类型。 -- TDengine 只接收 JSON **数组格式**的字符串,即使一行数据也需要转换成数组形式。 - 默认生成的子表名是根据规则生成的唯一 ID 值。用户也可以通过在 taos.cfg 里配置 smlChildTableName 参数来指定某个标签值作为子表名。该标签值应该具有全局唯一性。举例如下:假设有个标签名为tname, 配置 smlChildTableName=tname, 插入数据为 `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` 则创建的子表名为 cpu1。注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。 ::: diff --git a/docs/zh/07-develop/03-insert-data/_py_kafka.mdx b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx index cd7edf557d4ae41df0f55f4456fe405515132e51..d656325674aae33c3bd222fddeb5efb776593d59 100644 --- a/docs/zh/07-develop/03-insert-data/_py_kafka.mdx +++ b/docs/zh/07-develop/03-insert-data/_py_kafka.mdx @@ -55,6 +55,70 @@ for p in ps: ### 完整示例 +
+kafka_example_perform + +`kafka_example_perform` 是示例程序的入口 + +```py +{{#include docs/examples/python/kafka_example_perform.py}} +``` +
+ +
+kafka_example_common + +`kafka_example_common` 是示例程序的公共代码 + ```py -{{#include docs/examples/python/kafka_example.py}} +{{#include docs/examples/python/kafka_example_common.py}} ``` +
+ +
+kafka_example_producer + +`kafka_example_producer` 是示例程序的 producer 代码,负责生成并发送测试数据到 kafka + +```py +{{#include docs/examples/python/kafka_example_producer.py}} +``` +
+ +
+kafka_example_consumer + +`kafka_example_consumer` 是示例程序的 consumer 代码,负责从 kafka 消费数据,并写入到 TDengine + +```py +{{#include docs/examples/python/kafka_example_consumer.py}} +``` +
+ +### 执行步骤 + +
+ 执行 Python 示例程序 + + 1. 安装并启动 kafka + + 2. python 环境准备 + - 安装 python3 + - 安装 taospy + - 安装 kafka-python + + 3. 执行示例程序 + + 程序的执行入口是 `kafka_example_perform.py`,获取程序完整的执行参数,请执行 help 命令。 + + ``` + python3 kafka_example_perform.py --help + ``` + + 以下为创建 100 个子表,每个子表 20000 条数据,kafka max poll 为 100,一个进程,每个进程一个处理线程的程序执行命令 + + ``` + python3 kafka_example_perform.py -table-count=100 -table-items=20000 -max-poll=100 -threads=1 -processes=1 + ``` + +
diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 1f5a089aaa2a051e238eedc0315c37cad643b33f..fb171042d973ec8201ca0ae5f016c5d5b4324e85 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -92,22 +92,21 @@ void close() throws SQLException; ```python -class TaosConsumer(): - def __init__(self, *topics, **configs) +class Consumer: + def subscribe(self, topics): + pass - def __iter__(self) + def unsubscribe(self): + pass - def __next__(self) + def poll(self, timeout: float = 1.0): + pass - def sync_next(self) - - def subscription(self) + def close(self): + pass - def unsubscribe(self) - - def close(self) - - def __del__(self) + def commit(self, message): + pass ``` @@ -115,19 +114,22 @@ class TaosConsumer(): ```go -func NewConsumer(conf *Config) (*Consumer, error) - -func (c *Consumer) Close() error +func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error) -func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error +// 出于兼容目的保留 rebalanceCb 参数,当前未使用 +func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error -func (c *Consumer) FreeMessage(message unsafe.Pointer) +// 出于兼容目的保留 rebalanceCb 参数,当前未使用 +func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error -func (c *Consumer) Poll(timeout time.Duration) (*Result, error) +func (c *Consumer) Poll(timeoutMs int) tmq.Event -func (c *Consumer) Subscribe(topics []string) error +// 出于兼容目的保留 tmq.TopicPartition 参数,当前未使用 +func (c *Consumer) Commit() ([]tmq.TopicPartition, error) func (c *Consumer) Unsubscribe() error + +func (c *Consumer) Close() error ``` @@ -355,50 +357,20 @@ public class MetersDeserializer extends ReferenceDeserializer { ```go -config := tmq.NewConfig() -defer config.Destroy() -err = config.SetGroupID("test") -if err != nil { - panic(err) -} -err = config.SetAutoOffsetReset("earliest") -if err != nil { - panic(err) -} -err = config.SetConnectIP("127.0.0.1") -if err != nil { - panic(err) -} -err = config.SetConnectUser("root") -if err != nil { - panic(err) -} -err = config.SetConnectPass("taosdata") -if err != nil { - panic(err) -} -err = config.SetConnectPort("6030") -if err != nil { - panic(err) -} -err = config.SetMsgWithTableName(true) -if err != nil { - panic(err) -} -err = config.EnableHeartBeat() -if err != nil { - panic(err) -} -err = config.EnableAutoCommit(func(result *wrapper.TMQCommitCallbackResult) { - if result.ErrCode != 0 { - errStr := wrapper.TMQErr2Str(result.ErrCode) - err := errors.NewError(int(result.ErrCode), errStr) - panic(err) - } -}) -if err != nil { - panic(err) +conf := &tmq.ConfigMap{ + "group.id": "test", + "auto.offset.reset": "earliest", + "td.connect.ip": "127.0.0.1", + "td.connect.user": "root", + "td.connect.pass": "taosdata", + "td.connect.port": "6030", + "client.id": "test_tmq_c", + "enable.auto.commit": "false", + "enable.heartbeat.background": "true", + "experimental.snapshot.enable": "true", + "msg.with.table.name": "true", } +consumer, err := NewConsumer(conf) ``` @@ -420,34 +392,33 @@ let mut consumer = tmq.build()?; -Python 语言下引入 `taos` 库的 `TaosConsumer` 类,创建一个 Consumer 示例: +Python 语言下引入 `taos` 库的 `Consumer` 类,创建一个 Consumer 示例: ```python -from taos.tmq import TaosConsumer +from taos.tmq import Consumer -# Syntax: `consumer = TaosConsumer(*topics, **args)` +# Syntax: `consumer = Consumer(configs)` # # Example: -consumer = TaosConsumer('topic1', 'topic2', td_connect_ip = "127.0.0.1", group_id = "local") +consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) ``` -其中,元组类型参数被视为 *Topics*,字典类型参数用于以下订阅配置设置: +其中,`configs` 为 dict 类型,传递创建 Consumer 的参数。可以配置的参数有: -| 参数名称 | 类型 | 参数说明 | 备注 | -| :----------------------------: | :----: | -------------------------------------------------------- | ------------------------------------------- | -| `td_connect_ip` | string | 用于创建连接,同 `taos_connect` | | -| `td_connect_user` | string | 用于创建连接,同 `taos_connect` | | -| `td_connect_pass` | string | 用于创建连接,同 `taos_connect` | | -| `td_connect_port` | string | 用于创建连接,同 `taos_connect` | | -| `group_id` | string | 消费组 ID,同一消费组共享消费进度 | **必填项**。最大长度:192。 | -| `client_id` | string | 客户端 ID | 最大长度:192。 | -| `auto_offset_reset` | string | 消费组订阅的初始位置 | 可选:`earliest`(default), `latest`, `none` | -| `enable_auto_commit` | string | 启用自动提交 | 合法值:`true`, `false`,默认为 true | -| `auto_commit_interval_ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值:5000 ms | -| `enable_heartbeat_background` | string | 启用后台心跳,启用后即使长时间不 poll 消息也不会造成离线 | 合法值:`true`, `false` | -| `experimental_snapshot_enable` | string | 是否允许从 TSDB 消费数据 | 合法值:`true`, `false` | -| `msg_with_table_name` | string | 是否允许从消息中解析表名,不适用于列订阅 | 合法值:`true`, `false` | -| `timeout` | int | 消费者拉取数据的超时时间 | | +| 参数名称 | 类型 | 参数说明 | 备注 | +|:------:|:----:|:-------:|:---:| +| `td.connect.ip` | string | 用于创建连接|| +| `td.connect.user` | string | 用于创建连接|| +| `td.connect.pass` | string | 用于创建连接|| +| `td.connect.port` | string | 用于创建连接|| +| `group.id` | string | 消费组 ID,同一消费组共享消费进度 | **必填项**。最大长度:192 | +| `client.id` | string | 客户端 ID | 最大长度:192 | +| `msg.with.table.name` | string | 是否允许从消息中解析表名,不适用于列订阅 | 合法值:`true`, `false` | +| `enable.auto.commit` | string | 启用自动提交 | 合法值:`true`, `false` | +| `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值:5000 ms | +| `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选:`earliest`(default), `latest`, `none` | +| `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值:`true`, `false` | +| `enable.heartbeat.background` | string | 启用后台心跳,启用后即使长时间不 poll 消息也不会造成离线 | 合法值:`true`, `false` | @@ -532,11 +503,7 @@ consumer.subscribe(topics); ```go -consumer, err := tmq.NewConsumer(config) -if err != nil { - panic(err) -} -err = consumer.Subscribe([]string{"example_tmq_topic"}) +err = consumer.Subscribe("example_tmq_topic", nil) if err != nil { panic(err) } @@ -554,7 +521,7 @@ consumer.subscribe(["tmq_meters"]).await?; ```python -consumer = TaosConsumer('topic_ctb_column', group_id='vg2') +consumer.subscribe(['topic1', 'topic2']) ``` @@ -620,13 +587,17 @@ while(running){ ```go for { - result, err := consumer.Poll(time.Second) - if err != nil { - panic(err) + ev := consumer.Poll(0) + if ev != nil { + switch e := ev.(type) { + case *tmqcommon.DataMessage: + fmt.Println(e.Value()) + case tmqcommon.Error: + fmt.Fprintf(os.Stderr, "%% Error: %v: %v\n", e.Code(), e) + panic(e) + } + consumer.Commit() } - fmt.Println(result) - consumer.Commit(context.Background(), result.Message) - consumer.FreeMessage(result.Message) } ``` @@ -669,9 +640,17 @@ for { ```python -for msg in consumer: - for row in msg: - print(row) +while True: + res = consumer.poll(100) + if not res: + continue + err = res.error() + if err is not None: + raise err + val = res.value() + + for block in val: + print(block.fetchall()) ``` @@ -738,7 +717,11 @@ consumer.close(); ```go -consumer.Close() +/* Unsubscribe */ +_ = consumer.Unsubscribe() + +/* Close consumer */ +_ = consumer.Close() ``` diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx index e7de158cc8d2b0b686b25bbe96e7a092c2a68e51..a0505dabad08dd4a76f67139cde3125da8f830b3 100644 --- a/docs/zh/07-develop/_sub_java.mdx +++ b/docs/zh/07-develop/_sub_java.mdx @@ -1,3 +1,9 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + ```java {{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}} ``` @@ -6,4 +12,20 @@ ``` ```java {{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} -``` \ No newline at end of file +``` + + + + +```java +{{#include docs/examples/java/src/main/java/com/taos/example/WebsocketSubscribeDemo.java}} +``` +```java +{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} +``` +```java +{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} +``` + + + diff --git a/docs/zh/08-connector/05-schemaless-api.mdx b/docs/zh/08-connector/05-schemaless-api.mdx index f6d7e09212c71ed7b142eeabf2a2d69c5f79d439..8f0a9273b9988c191681063adfb67ebfd03615d3 100644 --- a/docs/zh/08-connector/05-schemaless-api.mdx +++ b/docs/zh/08-connector/05-schemaless-api.mdx @@ -19,6 +19,7 @@ TDengine 提供了兼容 InfluxDB (v1) 和 OpenTSDB 行协议的 Schemaless API - `precision` TDengine 使用的时间精度 - `u` TDengine 用户名 - `p` TDengine 密码 +- `ttl` 自动创建的子表生命周期,以子表的第一条数据的 TTL 参数为准,不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数 注意: 目前不支持 InfluxDB 的 token 验证方式,仅支持 Basic 验证和查询参数验证。 diff --git a/docs/zh/08-connector/10-cpp.mdx b/docs/zh/08-connector/10-cpp.mdx index ad83aee734e03b5bea807959c36ee90be4e1731d..35d30303c2b2b70da270cc0f49a8e867d8d43e99 100644 --- a/docs/zh/08-connector/10-cpp.mdx +++ b/docs/zh/08-connector/10-cpp.mdx @@ -330,6 +330,7 @@ typedef struct taosField { :::note 2.0 及以上版本 TDengine 推荐数据库应用的每个线程都建立一个独立的连接,或基于线程建立连接池。而不推荐在应用中将该连接 (TAOS\*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性,但 “USE statement” 等状态量有可能在线程之间相互干扰。此外,C 语言的连接器可以按照需求动态建立面向数据库的新连接(该过程对用户不可见),同时建议只有在程序最后退出的时候才调用 `taos_close()` 关闭连接。 +另一个需要注意的是,在上述同步 API 执行过程中,不能调用类似 pthread_cancel 之类的 API 来强制结束线程,因为涉及一些模块的同步操作,如果强制结束线程有可能造成包括但不限于死锁等异常状况。 ::: diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 1ee59d2df429f2bd64b1dfdcc2ef81f12b1272d0..061475f51e0cf9c25ba09d670e9ceef0bd2bab71 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -68,39 +68,38 @@ TDengine 目前支持时间戳、数字、字符、布尔类型,与 Java 对 ### 安装连接器 - + - 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - 仓库,且各大仓库都已同步。 +目前 taos-jdbcdriver 已经发布到 [Sonatype Repository](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) 仓库,且各大仓库都已同步。 - - [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) - - [maven.aliyun](https://maven.aliyun.com/mvn/search) +- [sonatype](https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [mvnrepository](https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver) +- [maven.aliyun](https://maven.aliyun.com/mvn/search) - Maven 项目中,在 pom.xml 中添加以下依赖: +Maven 项目中,在 pom.xml 中添加以下依赖: - ```xml-dtd - - com.taosdata.jdbc - taos-jdbcdriver - 3.0.0 - - ``` +```xml-dtd + + com.taosdata.jdbc + taos-jdbcdriver + 3.0.0 + +``` - - + + - 可以通过下载 TDengine 的源码,自己编译最新版本的 Java connector +可以通过下载 TDengine 的源码,自己编译最新版本的 Java connector - ```shell - git clone https://github.com/taosdata/taos-connector-jdbc.git - cd taos-connector-jdbc - mvn clean install -Dmaven.test.skip=true - ``` +```shell +git clone https://github.com/taosdata/taos-connector-jdbc.git +cd taos-connector-jdbc +mvn clean install -Dmaven.test.skip=true +``` - 编译后,在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包,并自动将编译的 jar 文件放在本地的 Maven 仓库中。 +编译后,在 target 目录下会产生 taos-jdbcdriver-3.0.*-dist.jar 的 jar 包,并自动将编译的 jar 文件放在本地的 Maven 仓库中。 - + ## 建立连接 @@ -111,125 +110,117 @@ TDengine 的 JDBC URL 规范格式为: 对于建立连接,原生连接与 REST 连接有细微不同。 - + - ```java - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.TSDBDriver"); +String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例,使用了 JDBC 原生连接的 TSDBDriver,建立了到 hostname 为 taosdemo.com,端口为 6030(TDengine 的默认端口),数据库名为 test 的连接。这个 URL - 中指定用户名(user)为 root,密码(password)为 taosdata。 +以上示例,使用了 JDBC 原生连接的 TSDBDriver,建立了到 hostname 为 taosdemo.com,端口为 6030(TDengine 的默认端口),数据库名为 test 的连接。这个 URL +中指定用户名(user)为 root,密码(password)为 taosdata。 - **注意**:使用 JDBC 原生连接,taos-jdbcdriver 需要依赖客户端驱动(Linux 下是 libtaos.so;Windows 下是 taos.dll;macOS 下是 libtaos.dylib)。 +**注意**:使用 JDBC 原生连接,taos-jdbcdriver 需要依赖客户端驱动(Linux 下是 libtaos.so;Windows 下是 taos.dll;macOS 下是 libtaos.dylib)。 - url 中的配置参数如下: +url 中的配置参数如下: - - user:登录 TDengine 用户名,默认值 'root'。 - - password:用户登录密码,默认值 'taosdata'。 - - cfgdir:客户端配置文件目录路径,Linux OS 上默认值 `/etc/taos`,Windows OS 上默认值 `C:/TDengine/cfg`。 - - charset:客户端使用的字符集,默认值为系统字符集。 - - locale:客户端语言环境,默认值系统当前 locale。 - - timezone:客户端使用的时区,默认值为系统当前时区。 - - batchfetch: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 - - batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false:不再执行失败 SQL - 后的任何语句。默认值为:false。 +- user:登录 TDengine 用户名,默认值 'root'。 +- password:用户登录密码,默认值 'taosdata'。 +- cfgdir:客户端配置文件目录路径,Linux OS 上默认值 `/etc/taos`,Windows OS 上默认值 `C:/TDengine/cfg`。 +- charset:客户端使用的字符集,默认值为系统字符集。 +- locale:客户端语言环境,默认值系统当前 locale。 +- timezone:客户端使用的时区,默认值为系统当前时区。 +- batchfetch: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:true。开启批量拉取同时获取一批数据在查询数据量较大时批量拉取可以有效的提升查询性能。 +- batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败将继续执行下面的 SQL。false:不再执行失败 SQL 后的任何语句。默认值为:false。 - JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 +JDBC 原生连接的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 - **使用 TDengine 客户端驱动配置文件建立连接 ** +**使用 TDengine 客户端驱动配置文件建立连接 ** - 当使用 JDBC 原生连接连接 TDengine 集群时,可以使用 TDengine 客户端驱动配置文件,在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示: +当使用 JDBC 原生连接连接 TDengine 集群时,可以使用 TDengine 客户端驱动配置文件,在配置文件中指定集群的 firstEp、secondEp 等参数。如下所示: - 1. 在 Java 应用中不指定 hostname 和 port +1. 在 Java 应用中不指定 hostname 和 port - ```java - public Connection getConn() throws Exception{ - Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; - Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - Connection conn = DriverManager.getConnection(jdbcUrl, connProps); - return conn; - } - ``` +```java +public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; +} +``` - 2. 在配置文件中指定 firstEp 和 secondEp +2. 在配置文件中指定 firstEp 和 secondEp - ```shell - # first fully qualified domain name (FQDN) for TDengine system - firstEp cluster_node1:6030 +```shell +# first fully qualified domain name (FQDN) for TDengine system +firstEp cluster_node1:6030 - # second fully qualified domain name (FQDN) for TDengine system, for cluster only - secondEp cluster_node2:6030 +# second fully qualified domain name (FQDN) for TDengine system, for cluster only +secondEp cluster_node2:6030 - # default system charset - # charset UTF-8 +# default system charset +# charset UTF-8 - # system locale - # locale en_US.UTF-8 - ``` +# system locale +# locale en_US.UTF-8 +``` - 以上示例,jdbc 会使用客户端的配置文件,建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时,JDBC 会尝试使用 secondEp - 连接集群。 +以上示例,jdbc 会使用客户端的配置文件,建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时,JDBC 会尝试使用 secondEp 连接集群。 - TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可以正常建立到集群的连接。 +TDengine 中,只要保证 firstEp 和 secondEp 中一个节点有效,就可以正常建立到集群的连接。 - > **注意**:这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 - C://TDengine/cfg/taos.cfg。 +> **注意**:这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件,Linux OS 上默认值 /etc/taos/taos.cfg ,Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 - - + + - ```java - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; - Connection conn = DriverManager.getConnection(jdbcUrl); - ``` +```java +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` - 以上示例,使用了 JDBC REST 连接的 RestfulDriver,建立了到 hostname 为 taosdemo.com,端口为 6041,数据库名为 test 的连接。这个 URL 中指定用户名(user)为 - root,密码(password)为 taosdata。 +以上示例,使用了 JDBC REST 连接的 RestfulDriver,建立了到 hostname 为 taosdemo.com,端口为 6041,数据库名为 test 的连接。这个 URL 中指定用户名(user)为 root,密码(password)为 taosdata。 - 使用 JDBC REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要: +使用 JDBC REST 连接,不需要依赖客户端驱动。与 JDBC 原生连接相比,仅需要: - 1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”; - 2. jdbcUrl 以“jdbc:TAOS-RS://”开头; - 3. 使用 6041 作为连接端口。 +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”; +2. jdbcUrl 以“jdbc:TAOS-RS://”开头; +3. 使用 6041 作为连接端口。 - url 中的配置参数如下: +url 中的配置参数如下: - - user:登录 TDengine 用户名,默认值 'root'。 - - password:用户登录密码,默认值 'taosdata'。 - - batchfetch: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST - 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP,WebSocket 可以使 JDBC REST 连接支持大数据量查询,并提升查询性能。 - - charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。 - - batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false:不再执行失败 SQL - 后的任何语句。默认值为:false。 - - httpConnectTimeout: 连接超时时间,单位 ms, 默认值为 5000。 - - httpSocketTimeout: socket 超时时间,单位 ms,默认值为 5000。仅在 batchfetch 设置为 false 时生效。 - - messageWaitTimeout: 消息超时时间, 单位 ms, 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 - - useSSL: 连接中是否使用 SSL。 +- user:登录 TDengine 用户名,默认值 'root'。 +- password:用户登录密码,默认值 'taosdata'。 +- batchfetch: true:在执行查询时批量拉取结果集;false:逐行拉取结果集。默认值为:false。逐行拉取结果集使用 HTTP 方式进行数据传输。JDBC REST 连接支持批量拉取数据功能。taos-jdbcdriver 与 TDengine 之间通过 WebSocket 连接进行数据传输。相较于 HTTP,WebSocket 可以使 JDBC REST 连接支持大数据量查询,并提升查询性能。 +- charset: 当开启批量拉取数据时,指定解析字符串数据的字符集。 +- batchErrorIgnore:true:在执行 Statement 的 executeBatch 时,如果中间有一条 SQL 执行失败,继续执行下面的 SQL 了。false:不再执行失败 SQL 后的任何语句。默认值为:false。 +- httpConnectTimeout: 连接超时时间,单位 ms, 默认值为 5000。 +- httpSocketTimeout: socket 超时时间,单位 ms,默认值为 5000。仅在 batchfetch 设置为 false 时生效。 +- messageWaitTimeout: 消息超时时间, 单位 ms, 默认值为 3000。 仅在 batchfetch 设置为 true 时生效。 +- useSSL: 连接中是否使用 SSL。 - **注意**:部分配置项(比如:locale、timezone)在 REST 连接中不生效。 +**注意**:部分配置项(比如:locale、timezone)在 REST 连接中不生效。 - :::note +:::note - - 与原生连接方式不同,REST 接口是无状态的。在使用 JDBC REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。例如: +- 与原生连接方式不同,REST 接口是无状态的。在使用 JDBC REST 连接时,需要在 SQL 中指定表、超级表的数据库名称。例如: - ```sql - INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); - ``` +```sql +INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFrancisco') VALUES(now, 24.6); +``` - - 如果在 url 中指定了 dbname,那么,JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url,在 SQL 中不需要指定 dbname。例如:url 为 - jdbc:TAOS-RS://127.0.0.1:6041/test,那么,可以执行 sql:insert into t1 using weather(ts, temperature) - tags('California.SanFrancisco') values(now, 24.6); +- 如果在 url 中指定了 dbname,那么,JDBC REST 连接会默认使用/rest/sql/dbname 作为 restful 请求的 url,在 SQL 中不需要指定 dbname。例如:url 为 jdbc:TAOS-RS://127.0.0.1:6041/test,那么,可以执行 sql:insert into t1 using weather(ts, temperature) tags('California.SanFrancisco') values(now, 24.6); - ::: +::: - + ### 指定 URL 和 Properties 获取连接 @@ -708,7 +699,10 @@ TaosConsumer consumer = new TaosConsumer<>(config); - enable.auto.commit: 是否允许自动提交。 - group.id: consumer: 所在的 group。 - value.deserializer: 结果集反序列化方法,可以继承 `com.taosdata.jdbc.tmq.ReferenceDeserializer`,并指定结果集 bean,实现反序列化。也可以继承 `com.taosdata.jdbc.tmq.Deserializer`,根据 SQL 的 resultSet 自定义反序列化方式。 -- 其他参数请参考:[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) +- td.connect.type: 连接方式。jni:表示使用动态库连接的方式,ws/WebSocket:表示使用 WebSocket 进行数据通信。默认为 jni 方式。 +- httpConnectTimeout:创建连接超时参数,单位 ms,默认为 5000 ms。仅在 WebSocket 连接下有效。 +- messageWaitTimeout:数据传输超时参数,单位 ms,默认为 10000 ms。仅在 WebSocket 连接下有效。 +其他参数请参考:[Consumer 参数列表](../../../develop/tmq#创建-consumer-以及consumer-group) #### 订阅消费数据 @@ -736,6 +730,9 @@ consumer.close() ### 使用示例如下: + + + ```java public abstract class ConsumerLoop { private final TaosConsumer consumer; @@ -807,6 +804,89 @@ public abstract class ConsumerLoop { } ``` + + + +除了原生的连接方式,Java 连接器还支持通过 WebSocket 订阅数据。 + +```java +public abstract class ConsumerLoop { + private final TaosConsumer consumer; + private final List topics; + private final AtomicBoolean shutdown; + private final CountDownLatch shutdownLatch; + + public ConsumerLoop() throws SQLException { + Properties config = new Properties(); + config.setProperty("bootstrap.servers", "localhost:6041"); + config.setProperty("td.connect.type", "ws"); + config.setProperty("msg.with.table.name", "true"); + config.setProperty("enable.auto.commit", "true"); + config.setProperty("group.id", "group2"); + config.setProperty("value.deserializer", "com.taosdata.jdbc.tmq.ConsumerTest.ConsumerLoop$ResultDeserializer"); + + this.consumer = new TaosConsumer<>(config); + this.topics = Collections.singletonList("topic_speed"); + this.shutdown = new AtomicBoolean(false); + this.shutdownLatch = new CountDownLatch(1); + } + + public abstract void process(ResultBean result); + + public void pollData() throws SQLException { + try { + consumer.subscribe(topics); + + while (!shutdown.get()) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(100)); + for (ResultBean record : records) { + process(record); + } + } + consumer.unsubscribe(); + } finally { + consumer.close(); + shutdownLatch.countDown(); + } + } + + public void shutdown() throws InterruptedException { + shutdown.set(true); + shutdownLatch.await(); + } + + public static class ResultDeserializer extends ReferenceDeserializer { + + } + + public static class ResultBean { + private Timestamp ts; + private int speed; + + public Timestamp getTs() { + return ts; + } + + public void setTs(Timestamp ts) { + this.ts = ts; + } + + public int getSpeed() { + return speed; + } + + public void setSpeed(int speed) { + this.speed = speed; + } + } +} +``` + + + + +> **注意**:这里的 value.deserializer 配置参数值应该根据测试环境的包路径做相应的调整。 + ### 与连接池使用 #### HikariCP @@ -890,8 +970,10 @@ public static void main(String[] args) throws Exception { | taos-jdbcdriver 版本 | 主要变化 | | :------------------: | :----------------------------: | -| 3.0.1 - 3.0.2 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译,JDK 8 环境下建议使用 3.0.2 版本 | +| 3.1.0 | WebSocket 连接支持订阅功能 | +| 3.0.1 - 3.0.4 | 修复一些情况下结果集数据解析错误的问题。3.0.1 在 JDK 11 环境编译,JDK 8 环境下建议使用其他版本 | | 3.0.0 | 支持 TDengine 3.0 | +| 2.0.42 | 修在 WebSocket 连接中 wasNull 接口返回值 | | 2.0.41 | 修正 REST 连接中用户名和密码转码方式 | | 2.0.39 - 2.0.40 | 增加 REST 连接/请求 超时设置 | | 2.0.38 | JDBC REST 连接增加批量拉取功能 | @@ -928,7 +1010,7 @@ public static void main(String[] args) throws Exception { **原因**:taos-jdbcdriver 3.0.1 版本需要在 JDK 11+ 环境使用。 -**解决方法**: 更换 taos-jdbcdriver 3.0.2 版本。 +**解决方法**: 更换 taos-jdbcdriver 3.0.2+ 版本。 其它问题请参考 [FAQ](../../../train-faq/faq) diff --git a/docs/zh/08-connector/20-go.mdx b/docs/zh/08-connector/20-go.mdx index 0fc4007f6362697222b425c8c2c803b911b9ac8a..2aa1a58e49f34b412f12bd0d67586dc6e56cf0bc 100644 --- a/docs/zh/08-connector/20-go.mdx +++ b/docs/zh/08-connector/20-go.mdx @@ -15,7 +15,7 @@ import GoOpenTSDBTelnet from "../07-develop/03-insert-data/_go_opts_telnet.mdx" import GoOpenTSDBJson from "../07-develop/03-insert-data/_go_opts_json.mdx" import GoQuery from "../07-develop/04-query-data/_go.mdx" -`driver-go` 是 TDengine 的官方 Go 语言连接器,实现了 Go 语言[ database/sql ](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 +`driver-go` 是 TDengine 的官方 Go 语言连接器,实现了 Go 语言 [database/sql](https://golang.org/pkg/database/sql/) 包的接口。Go 开发人员可以通过它开发存取 TDengine 集群数据的应用软件。 `driver-go` 提供两种建立连接的方式。一种是**原生连接**,它通过 TDengine 客户端驱动程序(taosc)原生连接 TDengine 运行实例,支持数据写入、查询、订阅、schemaless 接口和参数绑定接口等功能。另外一种是 **REST 连接**,它通过 taosAdapter 提供的 REST 接口连接 TDengine 运行实例。REST 连接实现的功能特性集合和原生连接有少量不同。 @@ -112,6 +112,7 @@ REST 连接支持所有能运行 Go 的平台。 ```text username:password@protocol(address)/dbname?param=value ``` + ### 使用连接器进行连接 @@ -176,6 +177,7 @@ func main() { } } ``` + @@ -207,6 +209,7 @@ func main() { } } ``` +
@@ -357,33 +360,32 @@ func main() { #### 订阅 -* `func NewConsumer(conf *Config) (*Consumer, error)` - -创建消费者。 +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` -* `func (c *Consumer) Subscribe(topics []string) error` + 创建消费者。 -订阅主题。 +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意:出于兼容目的保留 `rebalanceCb` 参数,当前未使用 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` + 订阅单个主题。 -轮询消息。 +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意:出于兼容目的保留 `rebalanceCb` 参数,当前未使用 -* `func (c *Consumer) Commit(ctx context.Context, message unsafe.Pointer) error` + 订阅主题。 -提交消息。 +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` -* `func (c *Consumer) FreeMessage(message unsafe.Pointer)` + 轮询消息。 -释放消息。 +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意:出于兼容目的保留 `tmq.TopicPartition` 参数,当前未使用 -* `func (c *Consumer) Unsubscribe() error` - -取消订阅。 + 提交消息。 * `func (c *Consumer) Close() error` -关闭消费者。 + 关闭连接。 #### schemaless @@ -443,25 +445,32 @@ func main() { ### 通过 WebSocket 订阅 -* `func NewConsumer(config *Config) (*Consumer, error)` +* `func NewConsumer(conf *tmq.ConfigMap) (*Consumer, error)` + + 创建消费者。 + +* `func (c *Consumer) Subscribe(topic string, rebalanceCb RebalanceCb) error` +注意:出于兼容目的保留 `rebalanceCb` 参数,当前未使用 - 创建消费者。 + 订阅单个主题。 -* `func (c *Consumer) Subscribe(topic []string) error` +* `func (c *Consumer) SubscribeTopics(topics []string, rebalanceCb RebalanceCb) error` +注意:出于兼容目的保留 `rebalanceCb` 参数,当前未使用 - 订阅主题。 + 订阅主题。 -* `func (c *Consumer) Poll(timeout time.Duration) (*Result, error)` +* `func (c *Consumer) Poll(timeoutMs int) tmq.Event` - 轮询消息。 + 轮询消息。 -* `func (c *Consumer) Commit(messageID uint64) error` +* `func (c *Consumer) Commit() ([]tmq.TopicPartition, error)` +注意:出于兼容目的保留 `tmq.TopicPartition` 参数,当前未使用 - 提交消息。 + 提交消息。 * `func (c *Consumer) Close() error` - 关闭消费者。 + 关闭连接。 完整订阅示例参见 [GitHub 示例文件](https://github.com/taosdata/driver-go/blob/3.0/examples/tmqoverws/main.go) diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 2ca11800c8b34dd8fe815cad8e81fe7b6faf79ad..1962df9607eb82ffaed75f9472a0c97fbc9f0ba3 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -32,7 +32,7 @@ Python 连接器的源码托管在 [GitHub](https://github.com/taosdata/taos-con ### 准备 -1. 安装 Python。建议使用 Python >= 3.7。如果系统上还没有 Python 可参考 [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) 安装。 +1. 安装 Python。新近版本 taospy 包要求 Python 3.6+。早期版本 taospy 包要求 Python 3.7+。taos-ws-py 包要求 Python 3.7+。如果系统上还没有 Python 可参考 [Python BeginnersGuide](https://wiki.python.org/moin/BeginnersGuide/Download) 安装。 2. 安装 [pip](https://pypi.org/project/pip/)。大部分情况下 Python 的安装包都自带了 pip 工具, 如果没有请参考 [pip documentation](https://pip.pypa.io/en/stable/installation/) 安装。 3. 如果使用原生连接,还需[安装客户端驱动](../#安装客户端驱动)。客户端软件包含了 TDengine 客户端动态链接库(libtaos.so 或 taos.dll) 和 TDengine CLI。 @@ -78,6 +78,23 @@ pip3 install git+https://github.com/taosdata/taos-connector-python.git +#### 安装 `taos-ws-py`(可选) + +taos-ws-py 包提供了通过 WebSocket 连接 TDengine 的能力,可选安装 taos-ws-py 以获得 WebSocket 连接 TDengine 的能力。 + + +##### 和 taospy 同时安装 + +```bash +pip3 install taospy[ws] +``` + +##### 单独安装 + +```bash +pip3 install taos-ws-py +``` + ### 安装验证 @@ -306,6 +323,30 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 数据订阅 + +连接器支持数据订阅功能,数据订阅功能请参考 [数据订阅](../../develop/tmq/)。 + + + + +`Consumer` 提供了 Python 连接器订阅 TMQ 数据的 API,相关 API 定义请参考 [数据订阅文档](../../develop/tmq/#%E4%B8%BB%E8%A6%81%E6%95%B0%E6%8D%AE%E7%BB%93%E6%9E%84%E5%92%8C-api)。 + +```python +{{#include docs/examples/python/tmq_example.py}} +``` + + + + +除了原生的连接方式,Python 连接器还支持通过 websocket 订阅 TMQ 数据。 + +```python +{{#include docs/examples/python/tmq_websocket_example.py}} +``` + + + ### 其它示例程序 | 示例程序链接 | 示例程序内容 | @@ -314,7 +355,7 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 | [bind_row.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/bind-row.py) | 参数绑定,一次绑定一行 | | [insert_lines.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/insert-lines.py) | InfluxDB 行协议写入 | | [json_tag.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/json-tag.py) | 使用 JSON 类型的标签 | -| [tmq.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq.py) | tmq 订阅 | +| [tmq_consumer.py](https://github.com/taosdata/taos-connector-python/blob/main/examples/tmq_consumer.py) | tmq 订阅 | ## 其它说明 diff --git a/docs/zh/08-connector/35-node.mdx b/docs/zh/08-connector/35-node.mdx index 35c28e3b9fc52698b813e65191732d3dd84c336e..25f8bdf1775357f4c81d11e37bfeb4f6f13a3d09 100644 --- a/docs/zh/08-connector/35-node.mdx +++ b/docs/zh/08-connector/35-node.mdx @@ -31,7 +31,8 @@ REST 连接器支持所有能运行 Node.js 的平台。 ## 支持的功能特性 -### 原生连接器 + + 1. 连接管理 2. 普通查询 @@ -40,12 +41,17 @@ REST 连接器支持所有能运行 Node.js 的平台。 5. 订阅功能 6. Schemaless -### REST 连接器 + + 1. 连接管理 2. 普通查询 3. 连续查询 + + + + ## 安装步骤 ### 安装前准备 @@ -62,6 +68,16 @@ REST 连接器支持所有能运行 Node.js 的平台。 - C 语言编译器,[GCC](https://gcc.gnu.org) v4.8.5 或更高版本 + + + +- `python` (建议`v2.7` , `v3.x.x` 目前还不支持) +- `@tdengine/client` 3.0.0 目前是只支持 Node.js v12.22.12 或 v12 的更高版本;其他版本可能存在包兼容性的问题 +- `make` +- C 语言编译器,[GCC](https://gcc.gnu.org) v4.8.5 或更高版本 + + + - 安装方法 1 @@ -104,6 +120,9 @@ npm install @tdengine/rest ### 安装验证 + + + 在安装好 TDengine 客户端后,使用 nodejsChecker.js 程序能够验证当前环境是否支持 Node.js 方式访问 TDengine。 验证方法: @@ -120,11 +139,35 @@ node nodejsChecker.js host=localhost - 执行以上步骤后,在命令行会输出 nodejsChecker.js 连接 TDengine 实例,并执行简单插入和查询的结果。 + + + +在安装好 TDengine 客户端后,使用 nodejsChecker.js 程序能够验证当前环境是否支持 Node.js 方式访问 TDengine。 + +验证方法: + +- 新建安装验证目录,例如:`~/tdengine-test`,下载 GitHub 上 [restChecker.js 源代码](https://github.com/taosdata/TDengine/tree/3.0/docs/examples/node/restexample/restChecker.js)到本地。 + +- 在命令行中执行以下命令。 + +```bash +npm init -y +npm install @tdengine/rest +node restChecker.js +``` + +- 执行以上步骤后,在命令行会输出 restChecker.js 连接 TDengine 实例,并执行简单插入和查询的结果。 + + + + + + ## 建立连接 请选择使用一种连接器。 - + 安装并引用 `@tdengine/client` 包。 @@ -171,24 +214,71 @@ let cursor = conn.cursor(); #### SQL 写入 + + + + + + +```js +{{#include docs/examples/node/restexample/insert_example.js}} +``` + + + + + + #### InfluxDB 行协议写入 + + + + + + #### OpenTSDB Telnet 行协议写入 + + + + + + #### OpenTSDB JSON 行协议写入 + + + + + + ### 查询数据 + + + + + + + +```js +{{#include docs/examples/node/restexample/query_example.js}} +``` + + + + ## 更多示例程序 diff --git a/docs/zh/08-connector/40-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx index a1a161d4eea9e2534ebb3a573211dcfae5dbb21f..80a831bab9d7343eaa85242c2e0e5e85c9f0d864 100644 --- a/docs/zh/08-connector/40-csharp.mdx +++ b/docs/zh/08-connector/40-csharp.mdx @@ -17,7 +17,7 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" `TDengine.Connector` 是 TDengine 提供的 C# 语言连接器。C# 开发人员可以通过它开发存取 TDengine 集群数据的 C# 应用软件。 -`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、数据订阅、schemaless 数据写入、参数绑定接口数据写入等功能。 `TDengine.Connector` 还支持 WebSocket,通过 DSN 建立 WebSocket 连接,提供数据写入、查询、参数绑定接口数据写入等功能。 +`TDengine.Connector` 连接器支持通过 TDengine 客户端驱动(taosc)建立与 TDengine 运行实例的连接,提供数据写入、查询、数据订阅、schemaless 数据写入、参数绑定接口数据写入等功能。 `TDengine.Connector` 自 v3.0.1 起还支持 WebSocket,通过 DSN 建立 WebSocket 连接,提供数据写入、查询、参数绑定接口数据写入等功能。 本文介绍如何在 Linux 或 Windows 环境中安装 `TDengine.Connector`,并通过 `TDengine.Connector` 连接 TDengine 集群,进行数据写入、查询等基本操作。 @@ -67,30 +67,45 @@ import CSAsyncQuery from "../07-develop/04-query-data/_cs_async.mdx" * [Nuget 客户端](https://docs.microsoft.com/en-us/nuget/install-nuget-client-tools) (可选安装) * 安装 TDengine 客户端驱动,具体步骤请参考[安装客户端驱动](../#安装客户端驱动) -### 使用 dotnet CLI 安装 +### 安装 TDengine.Connector - + -可以在当前 .NET 项目的路径下,通过 dotnet 命令引用 Nuget 中发布的 `TDengine.Connector` 到当前项目。 +可以在当前 .NET 项目的路径下,通过 dotnet CLI 添加 Nuget package `TDengine.Connector` 到当前项目。 ``` bash dotnet add package TDengine.Connector ``` +也可以修改当前项目的 `.csproj` 文件,添加如下 ItemGroup。 + +``` XML + + + +``` + - -也可以[下载源码](https://github.com/taosdata/taos-connector-dotnet/tree/3.0),直接引用 TDengine.Connector 库 + -```bash -git clone -b 3.0 https://github.com/taosdata/taos-connector-dotnet.git -cd taos-connector-dotnet -cp -r src/ myProject +需要修改目标项目的 `.csproj` 项目文件,将 `.nupkg` 中的 `runtimes` 目录中的动态库复制到当前项目的 `$(OutDir)` 目录下。 -cd myProject -dotnet add exmaple.csproj reference src/TDengine.csproj +```XML + + + + + + + + + ``` + +注意:`TDengine.Connector` 自 version>= 3.0.2 的 nuget package 中才会有动态库( taosws.dll,libtaows.so )。 + @@ -148,9 +163,9 @@ namespace TDengineExample 各部分意义见下表: -* **protocol**: 显示指定以何种方式建立连接,例如:`ws://localhost:6041` 指定以 Websocket 方式建立连接(支持http/ws)。 +* **protocol**: 显示指定以何种方式建立连接,例如:`ws://localhost:6041` 指定以 Websocket 方式建立连接(支持 http/ws )。 -* **username/password**: 用于创建连接的用户名及密码(默认`root/taosdata`)。 +* **username/password**: 用于创建连接的用户名及密码(默认 `root/taosdata` )。 * **host/port**: 指定创建连接的服务器及端口,WebSocket 连接默认为 `localhost:6041` 。 @@ -253,19 +268,20 @@ namespace TDengineExample |示例程序 | 示例程序描述 | |--------------------------------------------------------------------------------------------------------------------|--------------------------------------------| -| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | -| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | -| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | -| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | -| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | -| [数据订阅(TMQ)](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | -| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | -| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/5a4a7cd0dbcda114447cdc6d0c6dedd8e84a52da/examples/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | +| [CURD](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/Query/Query.cs) | 使用 TDengine.Connector 实现的建表、插入、查询示例 | +| [JSON Tag](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/JSONTag) | 使用 TDengine.Connector 实现的写入和查询 JSON tag 类型数据的示例 | +| [stmt](https://github.com/taosdata/taos-connector-dotnet/tree/3.0/examples/NET6Examples/Stmt) | 使用 TDengine.Connector 实现的参数绑定插入和查询的示例 | +| [schemaless](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/schemaless) | 使用 TDengine.Connector 实现的使用 schemaless 写入的示例 | +| [async query](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/AsyncQuery/QueryAsync.cs) | 使用 TDengine.Connector 实现的异步查询的示例 | +| [数据订阅(TMQ)](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/NET6Examples/TMQ/TMQ.cs) | 使用 TDengine.Connector 实现的订阅数据的示例 | +| [Basic WebSocket Usage](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSample.cs) | 使用 TDengine.Connector 的 WebSocket 基本的示例 | +| [Basic WebSocket STMT](https://github.com/taosdata/taos-connector-dotnet/blob/3.0/examples/FrameWork45/WS/WebSocketSTMT.cs) | 使用 TDengine.Connector 的 WebSocket STMT 基本的示例 | ## 重要更新记录 | TDengine.Connector | 说明 | |--------------------|--------------------------------| +| 3.0.2 | 支持 .NET Framework 4.5 及以上,支持 .NET standard 2.0。Nuget Package 包含 WebSocket 动态库。 | | 3.0.1 | 支持 WebSocket 和 Cloud,查询,插入,参数绑定。 | | 3.0.0 | 支持 TDengine 3.0.0.0,不兼容 2.x。新增接口TDengine.Impl.GetData(),解析查询结果。 | | 1.0.7 | 修复 TDengine.Query()内存泄露。 | diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index a28a7cd66c1cc7598d473903eb95df5192d933d2..f3f0f23b34d0b75b69d8dd866566ac98f306f13f 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -60,7 +60,7 @@ TDengine 版本更新往往会增加新的功能特性,列表中的连接器 | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | -| **数据订阅(TMQ)** | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | +| **数据订阅(TMQ)** | 暂不支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | | **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | | **批量拉取(基于 WebSocket)** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | diff --git a/docs/zh/10-deployment/05-helm.md b/docs/zh/10-deployment/05-helm.md index 9a3b21f09296e6f5a8dbd089225b6580b9567586..b2c405033f2814e3be5bcb298675e3bfb628babd 100644 --- a/docs/zh/10-deployment/05-helm.md +++ b/docs/zh/10-deployment/05-helm.md @@ -4,7 +4,7 @@ title: 使用 Helm 部署 TDengine 集群 description: 使用 Helm 部署 TDengine 集群的详细指南 --- -Helm 是 Kubernetes 的包管理器,上一节使用 Kubernets 部署 TDengine 集群的操作已经足够简单,但 Helm 依然可以提供更强大的能力。 +Helm 是 Kubernetes 的包管理器,上一节使用 Kubernetes 部署 TDengine 集群的操作已经足够简单,但 Helm 依然可以提供更强大的能力。 ## 安装 Helm @@ -23,7 +23,7 @@ Helm 会使用 kubectl 和 kubeconfig 的配置来操作 Kubernetes,可以参 TDengine Chart 尚未发布到 Helm 仓库,当前可以从 GitHub 直接下载: ```bash -wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.0.tgz +wget https://github.com/taosdata/TDengine-Operator/raw/3.0/helm/tdengine-3.0.2.tgz ``` @@ -39,7 +39,7 @@ kubectl get storageclass 之后,使用 helm 命令安装: ```bash -helm install tdengine tdengine-3.0.0.tgz \ +helm install tdengine tdengine-3.0.2.tgz \ --set storage.className= ``` @@ -47,7 +47,7 @@ helm install tdengine tdengine-3.0.0.tgz \ 在 minikube 环境下,可以设置一个较小的容量避免超出磁盘可用空间: ```bash -helm install tdengine tdengine-3.0.0.tgz \ +helm install tdengine tdengine-3.0.2.tgz \ --set storage.className=standard \ --set storage.dataSize=2Gi \ --set storage.logSize=10Mi @@ -84,14 +84,14 @@ TDengine 支持 `values.yaml` 自定义。 通过 helm show values 可以获取 TDengine Chart 支持的全部 values 列表: ```bash -helm show values tdengine-3.0.0.tgz +helm show values tdengine-3.0.2.tgz ``` 你可以将结果保存为 values.yaml,之后可以修改其中的各项参数,如 replica 数量,存储类名称,容量大小,TDengine 配置等,然后使用如下命令安装 TDengine 集群: ```bash -helm install tdengine tdengine-3.0.0.tgz -f values.yaml +helm install tdengine tdengine-3.0.2.tgz -f values.yaml ``` @@ -108,7 +108,7 @@ image: prefix: tdengine/tdengine #pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. -# tag: "3.0.0.0" +# tag: "3.0.2.0" service: # ClusterIP is the default service type, use NodeIP only if you know what you are doing. @@ -156,15 +156,15 @@ clusterDomainSuffix: "" # See the variable list at https://www.taosdata.com/cn/documentation/administrator . # # Note: -# 1. firstEp/secondEp: should not be setted here, it's auto generated at scale-up. -# 2. serverPort: should not be setted, we'll use the default 6030 in many places. -# 3. fqdn: will be auto generated in kubenetes, user should not care about it. +# 1. firstEp/secondEp: should not be set here, it's auto generated at scale-up. +# 2. serverPort: should not be set, we'll use the default 6030 in many places. +# 3. fqdn: will be auto generated in kubernetes, user should not care about it. # 4. role: currently role is not supported - every node is able to be mnode and vnode. # # Btw, keep quotes "" around the value like below, even the value will be number or not. taoscfg: # Starts as cluster or not, must be 0 or 1. - # 0: all pods will start as a seperate TDengine server + # 0: all pods will start as a separate TDengine server # 1: pods will start as TDengine server cluster. [default] CLUSTER: "1" diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 3918f240b4f9037db92af80df26dd471868b29de..d9ef3b51aa61acec3dfa87d86682033c85b6e455 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -27,10 +27,13 @@ database_option: { | PRECISION {'ms' | 'us' | 'ns'} | REPLICA value | RETENTIONS ingestion_duration:keep_duration ... - | STRICT {'off' | 'on'} | WAL_LEVEL {1 | 2} | VGROUPS value | SINGLE_STABLE {0 | 1} + | STT_TRIGGER value + | TABLE_PREFIX value + | TABLE_SUFFIX value + | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_ROLL_PERIOD value | WAL_RETENTION_SIZE value @@ -55,15 +58,12 @@ database_option: { - WAL_FSYNC_PERIOD:当 WAL 参数设置为 2 时,落盘的周期。默认为 3000,单位毫秒。最小为 0,表示每次写入立即落盘;最大为 180000,即三分钟。 - MAXROWS:文件块中记录的最大条数,默认为 4096 条。 - MINROWS:文件块中记录的最小条数,默认为 100 条。 -- KEEP:表示数据文件保存的天数,缺省值为 3650,取值范围 [1, 365000],且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式,如 KEEP 100h、KEEP 10d 等,支持 m(分钟)、h(小时)和 d(天)三个单位。也可以不写单位,如 KEEP 50,此时默认单位为天。 +- KEEP:表示数据文件保存的天数,缺省值为 3650,取值范围 [1, 365000],且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式,如 KEEP 100h、KEEP 10d 等,支持 m(分钟)、h(小时)和 d(天)三个单位。也可以不写单位,如 KEEP 50,此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间(多个以英文逗号分隔,最多 3 个,满足 keep 0 <= keep 1 <= keep 2,如 KEEP 100h,100d,3650d); 社区版不支持多级存储功能(即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间)。 - PAGES:一个 VNODE 中元数据存储引擎的缓存页个数,默认为 256,最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES,默认情况下为 1MB 内存。 - PAGESIZE:一个 VNODE 中元数据存储引擎的页大小,单位为 KB,默认为 4 KB。范围为 1 到 16384,即 1 KB 到 16 MB。 - PRECISION:数据库的时间戳精度。ms 表示毫秒,us 表示微秒,ns 表示纳秒,默认 ms 毫秒。 - REPLICA:表示数据库副本数,取值为 1 或 3,默认为 1。在集群中使用,副本数必须小于或等于 DNODE 的数目。 - RETENTIONS:表示数据的聚合周期和保存时长,如 RETENTIONS 15s:7d,1m:21d,15m:50d 表示数据原始采集周期为 15 秒,原始数据保存 7 天;按 1 分钟聚合的数据保存 21 天;按 15 分钟聚合的数据保存 50 天。目前支持且只支持三级存储周期。 -- STRICT:表示数据同步的一致性要求,默认为 off。 - - on 表示强一致,即运行标准的 raft 协议,半数提交返回成功。 - - off 表示弱一致,本地提交即返回成功。 - WAL_LEVEL:WAL 级别,默认为 1。 - 1:写 WAL,但不执行 fsync。 - 2:写 WAL,而且执行 fsync。 @@ -71,6 +71,10 @@ database_option: { - SINGLE_STABLE:表示此数据库中是否只可以创建一个超级表,用于超级表列非常多的情况。 - 0:表示可以创建多张超级表。 - 1:表示只可以创建一张超级表。 +- STT_TRIGGER:表示落盘文件触发文件合并的个数。默认为 1,范围 1 到 16。对于少表高频场景,此参数建议使用默认配置,或较小的值;而对于多表低频场景,此参数建议配置较大的值。 +- TABLE_PREFIX:内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度。 +- TABLE_SUFFIX:内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度。 +- TSDB_PAGESIZE:一个 VNODE 中时序数据存储引擎的页大小,单位为 KB,默认为 4 KB。范围为 1 到 16384,即 1 KB到 16 MB。 - WAL_RETENTION_PERIOD:wal 文件的额外保留策略,用于数据订阅。wal 的保存时长,单位为 s。单副本默认为 0,即落盘后立即删除。-1 表示不删除。多副本默认为 4 天。 - WAL_RETENTION_SIZE:wal 文件的额外保留策略,用于数据订阅。wal 的保存的最大上限,单位为 KB。单副本默认为 0,即落盘后立即删除。多副本默认为-1,表示不删除。 - WAL_ROLL_PERIOD:wal 文件切换时长,单位为 s。当 wal 文件创建并写入后,经过该时间,会自动创建一个新的 wal 文件。单副本默认为 0,即仅在落盘时创建新文件。多副本默认为 1 天。 @@ -112,12 +116,32 @@ alter_database_options: alter_database_option: { CACHEMODEL {'none' | 'last_row' | 'last_value' | 'both'} | CACHESIZE value + | BUFFER value + | PAGES value + | REPLICA value + | STT_TRIGGER value | WAL_LEVEL value | WAL_FSYNC_PERIOD value | KEEP value } ``` +### 修改 CACHESIZE + +修改数据库参数的命令使用简单,难的是如何确定是否需要修改以及如何修改。本小节描述如何判断数据库的 cachesize 是否够用。 + +1. 如何查看 cachesize? + +通过 select * from information_schema.ins_databases; 可以查看这些 cachesize 的具体值。 + +2. 如何查看 cacheload? + +通过 show .vgroups; 可以查看 cacheload + +3. 判断 cachesize 是否够用 + +如果 cacheload 非常接近 cachesize,则 cachesize 可能过小。 如果 cacheload 明显小于 cachesize 则 cachesize 是够用的。可以根据这个原则判断是否需要修改 cachesize 。具体修改值可以根据系统可用内存情况来决定是加倍或者是提高几倍。 + :::note 其它参数在 3.0.0.0 中暂不支持修改 @@ -134,7 +158,7 @@ SHOW DATABASES; ### 显示一个数据库的创建语句 ``` -SHOW CREATE DATABASE db_name; +SHOW CREATE DATABASE db_name \G; ``` 常用于数据库迁移。对一个已经存在的数据库,返回其创建语句;在另一个集群中执行该语句,就能得到一个设置完全相同的 Database。 @@ -142,7 +166,7 @@ SHOW CREATE DATABASE db_name; ### 查看数据库参数 ```sql -SELECT * FROM INFORMATION_SCHEMA.INS_DATABASES WHERE NAME='DBNAME' \G; +SELECT * FROM INFORMATION_SCHEMA.INS_DATABASES WHERE NAME='db_name' \G; ``` 会列出指定数据库的配置参数,并且每行只显示一个参数。 @@ -154,3 +178,19 @@ TRIM DATABASE db_name; ``` 删除过期数据,并根据多级存储的配置归整数据。 + +## 调整VGROUP中VNODE的分布 + +```sql +REDISTRIBUTE VGROUP vgroup_no DNODE dnode_id1 [DNODE dnode_id2] [DNODE dnode_id3] +``` + +按照给定的dnode列表,调整vgroup中的vnode分布。因为副本数目最大为3,所以最多输入3个dnode。 + +## 自动调整VGROUP中VNODE的分布 + +```sql +BALANCE VGROUP +``` + +自动调整集群所有vgroup中的vnode分布,相当于在vnode级别对集群进行数据的负载均衡操作。 diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 045abcde95e04cb4009dea9de597339c25fe36b0..5687c7e740766ad7a3fbea03ff42ce3137bb140e 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -57,7 +57,7 @@ table_option: { 3. MAX_DELAY:用于控制推送计算结果的最大延迟,默认值为 interval 的值(但不能超过最大值),最小单位毫秒,范围为 1 毫秒到 15 分钟,多个以逗号分隔。注:不建议 MAX_DELAY 设置太小,否则会过于频繁的推送结果,影响存储和查询性能,如无特殊需求,取默认值即可。只可用于超级表,且只有当数据库使用了 RETENTIONS 参数时,才可以使用此表参数。 4. ROLLUP:Rollup 指定的聚合函数,提供基于多层级的降采样聚合结果。只可用于超级表。只有当数据库使用了 RETENTIONS 参数时,才可以使用此表参数。作用于超级表除 TS 列外的其它所有列,但是只能定义一个聚合函数。 聚合函数支持 avg, sum, min, max, last, first。 5. SMA:Small Materialized Aggregates,提供基于数据块的自定义预计算功能。预计算类型包括 MAX、MIN 和 SUM。可用于超级表/普通表。 -6. TTL:Time to Live,是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数,当该表的存在时间超过 TTL 指定的时间后,TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间,系统不保证到了时间一定会将其删除,而只保证存在这样一个机制且最终一定会删除。TTL 单位是天,默认为 0,表示不限制,到期时间为表创建时间加上 TTL 时间。 +6. TTL:Time to Live,是用户用来指定表的生命周期的参数。如果创建表时指定了这个参数,当该表的存在时间超过 TTL 指定的时间后,TDengine 自动删除该表。这个 TTL 的时间只是一个大概时间,系统不保证到了时间一定会将其删除,而只保证存在这样一个机制且最终一定会删除。TTL 单位是天,默认为 0,表示不限制,到期时间为表创建时间加上 TTL 时间。TTL 与数据库 KEEP 参数没有关联,如果 KEEP 比 TTL 小,在表被删除之前数据也可能已经被删除。 ## 创建子表 diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index bd32da68aecc4c42be110e7716c1b2b4ebdab37c..c5933228decfb3eb865cad058fa6fbba77fefb58 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -139,10 +139,10 @@ alter_table_option: { - ADD COLUMN:添加列。 - DROP COLUMN:删除列。 -- MODIFY COLUMN:修改列定义,如果数据列的类型是可变长类型,那么可以使用此指令修改其宽度,只能改大,不能改小。 +- MODIFY COLUMN:修改列的宽度,数据列的类型必须是 nchar 和 binary,使用此指令可以修改其宽度,只能改大,不能改小。 - ADD TAG:给超级表添加一个标签。 - DROP TAG:删除超级表的一个标签。从超级表删除某个标签后,该超级表下的所有子表也会自动删除该标签。 -- MODIFY TAG:修改超级表的一个标签的定义。如果标签的类型是可变长类型,那么可以使用此指令修改其宽度,只能改大,不能改小。 +- MODIFY TAG:修改超级表的一个标签的列宽度。标签的类型只能是 nchar 和 binary,使用此指令可以修改其宽度,只能改大,不能改小。 - RENAME TAG:修改超级表的一个标签的名称。从超级表修改某个标签名后,该超级表下的所有子表也会自动更新该标签名。 ### 增加列 diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index ccb8f40b21f7f0082f7bf420643e88a618d83f0e..b72754b15437a081b89056fca3a76030cac62363 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -28,7 +28,7 @@ INSERT INTO tb_name [(field1_name, ...)] subquery 2. 时间戳不同的格式语法会有不同的精度影响。字符串格式的时间戳写法不受所在 DATABASE 的时间精度设置影响;而长整形格式的时间戳写法会受到所在 DATABASE 的时间精度设置影响。例如,时间戳"2021-07-13 16:16:48"的 UNIX 秒数为 1626164208。则其在毫秒精度下需要写作 1626164208000,在微秒精度设置下就需要写为 1626164208000000,纳秒精度设置下需要写为 1626164208000000000。 3. 一次插入多行数据时,不要把首列的时间戳的值都写 NOW。否则会导致语句中的多条记录使用相同的时间戳,于是就可能出现相互覆盖以致这些数据行无法全部被正确保存。其原因在于,NOW 函数在执行中会被解析为所在 SQL 语句的客户端执行时间,出现在同一语句中的多个 NOW 标记也就会被替换为完全相同的时间戳取值。 - 允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的 KEEP 值(数据保留的天数)。允许插入的最新记录的时间戳,是相对于当前服务器时间,加上配置的 DURATION 值(数据文件存储数据的时间跨度,单位为天)。KEEP 和 DURATION 都是可以在创建数据库时指定的,缺省值分别是 3650 天和 10 天。 + 允许插入的最老记录的时间戳,是相对于当前服务器时间,减去配置的 KEEP 值(数据保留的天数, 可以在创建数据库时指定,缺省值是 3650 天)。允许插入的最新记录的时间戳,取决于数据库的 PRECISION 值(时间戳精度, 可以在创建数据库时指定, ms 表示毫秒,us 表示微秒,ns 表示纳秒,默认毫秒):如果是毫秒或微秒, 取值为 1970 年 1 月 1 日 00:00:00.000 UTC 加上 1000 年, 即 2970 年 1 月 1 日 00:00:00.000 UTC; 如果是纳秒, 取值为 1970 年 1 月 1 日 00:00:00.000000000 UTC 加上 292 年, 即 2262 年 1 月 1 日 00:00:00.000000000 UTC。 **语法说明** diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index b9b26279e623f2f9f167c3c7f305f5921501f7b4..dcc085b3bd714116455badbdc268c41b717b818f 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -354,9 +354,9 @@ SELECT AVG(CASE WHEN voltage < 200 or voltage > 250 THEN 220 ELSE voltage END) F ## JOIN 子句 -TDengine 支持“普通表与普通表之间”、“超级表与超级表之间”、“子查询与子查询之间” 进行自然连接。自然连接与内连接的主要区别是,自然连接要求参与连接的字段在不同的表/超级表中必须是同名字段。也即,TDengine 在连接关系的表达中,要求必须使用同名数据列/标签列的相等关系。 +TDengine 支持基于时间戳主键的内连接,即 JOIN 条件必须包含时间戳主键。只要满足基于时间戳主键这个要求,普通表、子表、超级表和子查询之间可以随意的进行内连接,且对表个数没有限制。 -在普通表与普通表之间的 JOIN 操作中,只能使用主键时间戳之间的相等关系。例如: +普通表与普通表之间的 JOIN 操作: ```sql SELECT * @@ -364,7 +364,7 @@ FROM temp_tb_1 t1, pressure_tb_1 t2 WHERE t1.ts = t2.ts ``` -在超级表与超级表之间的 JOIN 操作中,除了主键时间戳一致的条件外,还要求引入能实现一一对应的标签列的相等关系。例如: +超级表与超级表之间的 JOIN 操作: ```sql SELECT * @@ -372,20 +372,15 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -类似地,也可以对多个子查询的查询结果进行 JOIN 操作。 - -:::note - -JOIN 语句存在如下限制要求: +子表与超级表之间的 JOIN 操作: -- 参与一条语句中 JOIN 操作的表/超级表最多可以有 10 个。 -- 在包含 JOIN 操作的查询语句中不支持 FILL。 -- 暂不支持参与 JOIN 操作的表之间聚合后的四则运算。 -- 不支持只对其中一部分表做 GROUP BY。 -- JOIN 查询的不同表的过滤条件之间不能为 OR。 -- JOIN 查询要求连接条件不能是普通列,只能针对标签和主时间字段列(第一列)。 +```sql +SELECT * +FROM temp_ctable t1, temp_stable t2 +WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; +``` -::: +类似地,也可以对多个子查询的查询结果进行 JOIN 操作。 ## 嵌套查询 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index c44c09d10c115a41efd22048d4fdafdc85370ac2..94f805205183aca6a7a08d9a836b0df33c7ed1f0 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -798,18 +798,22 @@ HISTOGRAM(expr,bin_type, bin_description, normalized) ### PERCENTILE ```sql -PERCENTILE(expr, p) +PERCENTILE(expr, p [, p1] ... ) ``` **功能说明**:统计表中某列的值百分比分位数。 -**返回数据类型**: DOUBLE。 +**返回数据类型**: 该函数最小参数个数为 2 个,最大参数个数为 11 个。可以最多同时返回 10 个百分比分位数。当参数个数为 2 时, 返回一个分位数, 类型为DOUBLE,当参数个数大于 2 时,返回类型为VARCHAR, 格式为包含多个返回值的JSON数组。 **应用字段**:数值类型。 **适用于**:表。 -**使用说明**:*P*值取值范围 0≤*P*≤100,为 0 的时候等同于 MIN,为 100 的时候等同于 MAX。 +**使用说明**: + +- *P*值取值范围 0≤*P*≤100,为 0 的时候等同于 MIN,为 100 的时候等同于 MAX; +- 同时计算针对同一列的多个分位数时,建议使用一个PERCENTILE函数和多个参数的方式,能很大程度上降低查询的响应时间。 + 比如,使用查询SELECT percentile(col, 90, 95, 99) FROM table, 性能会优于SELECT percentile(col, 90), percentile(col, 95), percentile(col, 99) from table。 ## 选择函数 @@ -879,7 +883,8 @@ INTERP(expr) - INTERP 根据 EVERY(time_unit) 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间(time_unit 值)进行插值,time_unit 可取值时间单位:1a(毫秒),1s(秒),1m(分),1h(小时),1d(天),1w(周)。例如 EVERY(500a) 将对于指定数据每500毫秒间隔进行一次插值. - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。关于 FILL 子句如何使用请参考 [FILL 子句](../distinguished/#fill-子句) - INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。 -- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 +- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.2.0版本以后支持)。 +- INTERP 可以与伪列 _isfilled 一起使用,显示返回结果是否为原始记录或插值算法产生的数据(3.0.3.0版本以后支持)。 ### LAST diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index a70d559a860c4e6ec44cbd13c34f7f306ab452cb..9e2fde526e1e63e3fbfb6124420caee3491cf965 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -114,7 +114,7 @@ SELECT * from information_schema.`ins_streams`; 在创建流时,可以通过 TRIGGER 指令指定流式计算的触发模式。 -对于非窗口计算,流式计算的触发是实时的;对于窗口计算,目前提供 3 种触发模式: +对于非窗口计算,流式计算的触发是实时的;对于窗口计算,目前提供 3 种触发模式,默认为 AT_ONCE: 1. AT_ONCE:写入立即触发 diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 8013698fced2f31ca21dfa220066b027e71cb856..8fd704ef551c9ebda61678bd129917e67cbfbef8 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -18,6 +18,7 @@ description: TDengine 保留关键字的详细列表 - ADD - AFTER - AGGREGATE +- ALIVE - ALL - ALTER - ANALYZE diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index 4bd1e52284f46d94cfd892c82a7fbe119b685532..901548e13270d07da0c9eaec19a0c011d8be189e 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -179,6 +179,83 @@ SHOW TABLE DISTRIBUTED table_name; 显示表的数据分布信息。 +示例说明: + +语句: show table distributed d0\G; 竖行显示表 d0 的 BLOCK 分布情况 + +
+ 显示示例 +

+
+*************************** 1.row ***************************
+
+_block_dist: Total_Blocks=[5] Total_Size=[93.65 Kb] Average_size=[18.73 Kb] Compression_Ratio=[23.98 %]
+
+Total_Blocks:  表 d0 占用的 block 个数为 5 个
+
+Total_Size:    表 d0 所有 block 在文件中占用的大小为 93.65 KB 
+
+Average_size:  平均每个 block 在文件中占用的空间大小为 18.73 KB
+
+Compression_Ratio: 数据压缩率 23.98%
+
+ 
+*************************** 2.row ***************************
+
+_block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000]
+
+Total_Rows:  统计表 d0 的所有行数 为20000 行(该数值仅供参考,不是精确的行数。获得精确的行数需要使用 count 函数)
+
+Inmem_Rows: 存储在写缓存中的数据行数(没有落盘),0 行表示内存缓存中没有数据
+
+MinRows:    BLOCK 中最小的行数,为 3616 行
+
+MaxRows:    BLOCK 中最大的行数,为 4096 行
+
+Average_Rows: 每个 BLOCK 中的平均行数,此时为 4000 行
+
+
+*************************** 3.row ***************************
+
+_block_dist: Total_Tables=[1] Total_Files=[2] Total_Vgroups=[1]
+
+Total_Tables:   子表的个数,这里为 1
+
+Total_Files:   表数据被分别保存的数据文件数量,这里是 2 个文件
+
+Total_Vgroups: 表数据分布的虚拟节点(vnode)数量
+
+
+*************************** 5.row ***************************
+
+_block_dist: 0100 |
+
+*************************** 6.row ***************************
+
+_block_dist: 0299 |
+
+......
+
+*************************** 22.row ***************************
+
+_block_dist: 3483 |||||||||||||||||  1 (20.00%)
+
+*************************** 23.row ***************************
+
+_block_dist: 3682 |
+
+*************************** 24.row ***************************
+
+_block_dist: 3881 |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||  4 (80.00%)
+
+Query OK, 24 row(s) in set (0.002444s)
+
+
+
+ + 上面是块中包含数据行数的块儿分布情况图,这里的 0100 0299 0498 … 表示的是每个块中包含的数据行数,上面的意思就是这个表的 5 个块,分布在 3483 ~3681 行的块有 1 个,占整个块的 20%,分布在 3881 ~ 4096(最大行数)的块数为 4 个,占整个块的 80%, 其它区域内分布块数为 0。 + + ## SHOW TAGS ```sql @@ -231,7 +308,7 @@ SHOW [db_name.]VGROUPS; ## SHOW VNODES ```sql -SHOW VNODES [dnode_name]; +SHOW VNODES {dnode_id | dnode_endpoint}; ``` 显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。 diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index 1ab7f4f0169932576edf9087a825367573713b03..9d67533cdea5c055c8d3ff58000fb80ac640e271 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -54,7 +54,6 @@ description: "TDengine 3.0 版本的语法变更说明" | 27 | GRANT | 新增 | 授予用户权限。 | 28 | KILL TRANSACTION | 新增 | 终止管理节点的事务。 | 29 | KILL STREAM | 废除 | 终止连续查询。3.0版本不再支持连续查询,而是用更通用的流计算来代替。 -| 30 | MERGE VGROUP | 新增 | 合并VGROUP。 | 31 | REVOKE | 新增 | 回收用户权限。 | 32 | SELECT | 调整 |
  • SELECT关闭隐式结果列,输出列均需要由SELECT子句来指定。
  • DISTINCT功能全面支持。2.x版本只支持对标签列去重,并且不可以和JOIN、GROUP BY等子句混用。
  • JOIN功能增强。增加支持:JOIN后WHERE条件中有OR条件;JOIN后的多表运算;JOIN后的多表GROUP BY。
  • FROM后子查询功能大幅增强。不限制子查询嵌套层数;支持子查询和UNION ALL混合使用;移除其他一些之前版本的语法限制。
  • WHERE后可以使用任意的标量表达式。
  • GROUP BY功能增强。支持任意标量表达式及其组合的分组。
  • SESSION可以用于超级表了。没有PARTITION BY时,超级表的数据会被合并成一条时间线。
  • STATE_WINDOW可以用于超级表了。没有PARTITION BY时,超级表的数据会被合并成一条时间线。
  • ORDER BY功能大幅增强。不再必须和GROUP BY子句一起使用;不再有排序表达式个数的限制;增加支持NULLS FIRST/LAST语法功能;支持符合语法语义的任意表达式。
  • 新增PARTITION BY语法。替代原来的GROUP BY tags。
| 33 | SHOW ACCOUNTS | 废除 | 2.x中为企业版功能,3.0不再支持。语法暂时保留了,执行报“This statement is no longer supported”错误。 @@ -76,8 +75,9 @@ description: "TDengine 3.0 版本的语法变更说明" | 49 | SHOW TRANSACTIONS | 新增 | 显示当前系统中正在执行的事务的信息。 | 50 | SHOW DNODE VARIABLES | 新增 |显示指定DNODE的配置参数。 | 51 | SHOW VNODES | 暂不支持 | 显示当前系统中VNODE的信息。3.0.0版本暂不支持。 -| 52 | SPLIT VGROUP | 新增 | 拆分VGROUP。 -| 53 | TRIM DATABASE | 新增 | 删除过期数据,并根据多级存储的配置归整数据。 +| 52 | TRIM DATABASE | 新增 | 删除过期数据,并根据多级存储的配置归整数据。 +| 53 | REDISTRIBUTE VGROUP | 新增 | 调整VGROUP中VNODE的分布。 +| 54 | BALANCE VGROUP | 新增 | 自动调整VGROUP中VNODE的分布。 ## SQL 函数变更 @@ -94,6 +94,7 @@ description: "TDengine 3.0 版本的语法变更说明" | 9 | SAMPLE | 增强 | 可以直接用于超级表了。没有PARTITION BY时,超级表的数据会被合并成一条时间线。 | 10 | STATECOUNT | 增强 | 可以直接用于超级表了。没有PARTITION BY时,超级表的数据会被合并成一条时间线。 | 11 | STATEDURATION | 增强 | 可以直接用于超级表了。没有PARTITION BY时,超级表的数据会被合并成一条时间线。 +| 12 | TIMETRUNCATE | 增强 | 增加ignore_timezone参数,可选是否使用,默认值为1. ## SCHEMALESS 变更 diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index 4e1e7b7ace09f7cb43e5049ebefe21a24525b595..5c155bdd6ebe9f1a4a38886f30b753cbda4ab6ab 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -21,6 +21,7 @@ taosAdapter 提供以下功能: - 无缝连接到 collectd - 无缝连接到 StatsD - 支持 Prometheus remote_read 和 remote_write +- 获取 table 所在的虚拟节点组(VGroup)的 VGroup ID ## taosAdapter 架构图 @@ -59,6 +60,7 @@ Usage of taosAdapter: --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" (default 0, means no ttl) -c, --config string config path default /etc/taos/taosadapter.toml --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" @@ -100,6 +102,7 @@ Usage of taosAdapter: --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL"(default 0, means no ttl) --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) @@ -110,6 +113,7 @@ Usage of taosAdapter: --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL"(default 0, means no ttl) --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" (default 1h0m0s) --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" (default 4000) --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" (default 4000) @@ -131,6 +135,7 @@ Usage of taosAdapter: --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" (default 0, means no ttl) --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" --version Print the version and exit ``` @@ -174,6 +179,7 @@ AllowWebSockets node_export 是一个机器指标的导出器。请访问 [https://github.com/prometheus/node_exporter](https://github.com/prometheus/node_exporter) 了解更多信息。 - 支持 Prometheus remote_read 和 remote_write remote_read 和 remote_write 是 Prometheus 数据读写分离的集群方案。请访问[https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/#remote-apis) 了解更多信息。 +- 获取 table 所在的虚拟节点组(VGroup)的 VGroup ID。关于虚拟节点组(VGroup)的更多信息,请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 ## 接口 @@ -195,6 +201,7 @@ AllowWebSockets - `precision` TDengine 使用的时间精度 - `u` TDengine 用户名 - `p` TDengine 密码 +- `ttl` 自动创建的子表生命周期,以子表的第一条数据的 TTL 参数为准,不可更新。更多信息请参考[创建表文档](taos-sql/table/#创建表)的 TTL 参数。 注意: 目前不支持 InfluxDB 的 token 验证方式,仅支持 Basic 验证和查询参数验证。 示例: curl --request POST http://127.0.0.1:6041/influxdb/v1/write?db=test --user "root:taosdata" --data-binary "measurement,host=host1 field1=2i,field2=2.0 1577836800000000000" @@ -235,6 +242,10 @@ Prometheus 使用的由 \*NIX 内核暴露的硬件和操作系统指标的输 +### 获取 table 的 VGroup ID + +可以访问 http 接口 `http://:6041/rest/vgid?db=&table=` 获取 table 的 VGroup ID。关于虚拟节点组(VGroup)的更多信息,请访问[整体架构文档](/tdinternal/arch/#主要逻辑单元) 。 + ## 内存使用优化方法 taosAdapter 将监测自身运行过程中内存使用率并通过两个阈值进行调节。有效值范围为 -1 到 100 的整数,单位为系统物理内存的百分比。 @@ -277,7 +288,7 @@ http 返回内容: ## taosAdapter 监控指标 -taosAdapter 采集 http 相关指标、cpu 百分比和内存百分比。 +taosAdapter 采集 http 相关指标、CPU 百分比和内存百分比。 ### http 接口 @@ -289,13 +300,13 @@ http://:6041/metrics ### 写入 TDengine -taosAdapter 支持将 http 监控、cpu 百分比和内存百分比写入 TDengine。 +taosAdapter 支持将 http 监控、CPU 百分比和内存百分比写入 TDengine。 有关配置参数 | **配置项** | **描述** | **默认值** | |-------------------------|--------------------------------------------|----------| -| monitor.collectDuration | cpu 和内存采集间隔 | 3s | +| monitor.collectDuration | CPU 和内存采集间隔 | 3s | | monitor.identity | 当前taosadapter 的标识符如果不设置将使用 'hostname:port' | | | monitor.incgroup | 是否是 cgroup 中运行(容器中运行设置为 true) | false | | monitor.writeToTD | 是否写入到 TDengine | false | diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index 76dd5f12d89b223292a8c868fd66135ec204319b..9091e71d1f9b118660d5f75ef9be5627143393b0 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -204,6 +204,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **-a/--replica ** : 创建数据库时指定其副本数,默认值为 1 。 +- ** -k/--keep-trying ** : 失败后进行重试的次数,默认不重试。需使用 v3.0.9 以上版本。 + +- ** -z/--trying-interval ** : 失败重试间隔时间,单位为毫秒,仅在 -k 指定重试后有效。需使用 v3.0.9 以上版本。 + - **-V/--version** : 显示版本信息并退出。不能与其它参数混用。 @@ -231,6 +235,10 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) 插入场景下 `filetype` 必须设置为 `insert`,该参数及其它通用参数详见[通用配置参数](#通用配置参数) +- ** keep_trying ** : 失败后进行重试的次数,默认不重试。需使用 v3.0.9 以上版本。 + +- ** trying_interval ** : 失败重试间隔时间,单位为毫秒,仅在 keep_trying 指定重试后有效。需使用 v3.0.9 以上版本。 + #### 数据库相关配置参数 创建数据库时的相关参数在 json 配置文件中的 `dbinfo` 中配置,个别具体参数如下。其余参数均与 TDengine 中 `create database` 时所指定的数据库参数相对应,详见[../../taos-sql/database] diff --git a/docs/zh/14-reference/06-taosdump.md b/docs/zh/14-reference/06-taosdump.md index 625499a94926ac3f86e4d34976c70a0bfe7b0954..8a031d147377713d0478d419798240f6f1194377 100644 --- a/docs/zh/14-reference/06-taosdump.md +++ b/docs/zh/14-reference/06-taosdump.md @@ -22,7 +22,7 @@ taosdump 是一个逻辑备份工具,它不应被用于备份任何原始数 taosdump 有两种安装方式: -- 安装 taosTools 官方安装包, 请从[所有下载链接](https://www.taosdata.com/all-downloads)页面找到 taosTools 并下载安装。 +- 安装 taosTools 官方安装包, 请从[发布历史页面](https://docs.taosdata.com/releases/tools/)页面找到 taosTools 并下载安装。 - 单独编译 taos-tools 并安装, 详情请参考 [taos-tools](https://github.com/taosdata/taos-tools) 仓库。 diff --git a/docs/zh/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp b/docs/zh/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp index 942130d4fabf7944c7add10acb3bb42ca7f51e0f..a90477880ff3a2cb2068540b55b69e4bbdbd8d4c 100644 Binary files a/docs/zh/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp and b/docs/zh/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp differ diff --git a/docs/zh/14-reference/07-tdinsight/index.mdx b/docs/zh/14-reference/07-tdinsight/index.mdx index 7f4c46a636ace97c6b04849d1b9805fcfc92da82..8ec11378ee7e96f8bea26cd8c7b79dcfe2557c12 100644 --- a/docs/zh/14-reference/07-tdinsight/index.mdx +++ b/docs/zh/14-reference/07-tdinsight/index.mdx @@ -345,11 +345,12 @@ TDinsight 仪表盘旨在提供 TDengine 相关资源的使用情况和状态, 支持监控 taosAdapter 请求统计和状态详情。包括: -1. **http_request_inflight**: 即时处理请求数 -2. **http_request_total**: 请求总数。 -3. **http_request_fail**: 请求总数。 -4. **CPU Used**: taosAdapter CPU 使用情况。 -5. **Memory Used**: taosAdapter 内存使用情况。 +1. **Http Request Total**: 请求总数。 +2. **Http Request Fail**: 请求总数。 +3. **CPU Used**: taosAdapter CPU 使用情况。 +4. **Memory Used**: taosAdapter 内存使用情况。 +5. **Http Request Inflight**: 即时处理请求数。 +6. **Http Status Code**: taosAdapter http 状态码。 ## 升级 diff --git a/docs/zh/14-reference/11-docker/index.md b/docs/zh/14-reference/11-docker/index.md index 086d72940c88adfe0e0c4c30cf16cc9cb2701630..a6696977f9cb08da2eb8ff68c42c54d388e74980 100644 --- a/docs/zh/14-reference/11-docker/index.md +++ b/docs/zh/14-reference/11-docker/index.md @@ -309,7 +309,7 @@ services: TAOS_FIRST_EP: "td-1" volumes: - taosdata-td3:/var/lib/taos/ - - taoslog-td3:/var/log/taos/ + - taoslog-td3:/var/log/taos/ volumes: taosdata-td1: taoslog-td1: @@ -473,18 +473,18 @@ Creating service taos_adapter ```shell $ docker stack ps taos ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -7m3sbf532bqp taos_adapter.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago -pj403n6ofmmh taos_adapter.2 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago -rxqfwsyk5q1h taos_adapter.3 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago -qj40lpxr40oc taos_adapter.4 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago -oe3455ulxpze taos_nginx.1 nginx:latest vm98 Running Running about a minute ago -o0tsg70nrrc6 taos_td-1.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +7m3sbf532bqp taos_adapter.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +pj403n6ofmmh taos_adapter.2 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +rxqfwsyk5q1h taos_adapter.3 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +qj40lpxr40oc taos_adapter.4 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago +oe3455ulxpze taos_nginx.1 nginx:latest vm98 Running Running about a minute ago +o0tsg70nrrc6 taos_td-1.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago q5m1oxs589cp taos_td-2.1 tdengine/tdengine:3.0.0.0 vm98 Running Running about a minute ago $ docker service ls ID NAME MODE REPLICAS IMAGE PORTS -ozuklorgl8bs taos_adapter replicated 4/4 tdengine/tdengine:3.0.0.0 +ozuklorgl8bs taos_adapter replicated 4/4 tdengine/tdengine:3.0.0.0 crmhdjw6vxw0 taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp -o86ngy7csv5n taos_td-1 replicated 1/1 tdengine/tdengine:3.0.0.0 +o86ngy7csv5n taos_td-1 replicated 1/1 tdengine/tdengine:3.0.0.0 rma040ny4tb0 taos_td-2 replicated 1/1 tdengine/tdengine:3.0.0.0 ``` @@ -495,11 +495,11 @@ rma040ny4tb0 taos_td-2 replicated 1/1 tdengine/tdengine:3.0.0.0 ```shell $ docker service scale taos_adapter=1 taos_adapter scaled to 1 -overall progress: 1 out of 1 tasks -1/1: running [==================================================>] +overall progress: 1 out of 1 tasks +1/1: running [==================================================>] verify: Service converged $ docker service ls -f name=taos_adapter ID NAME MODE REPLICAS IMAGE PORTS -ozuklorgl8bs taos_adapter replicated 1/1 tdengine/tdengine:3.0.0.0 +ozuklorgl8bs taos_adapter replicated 1/1 tdengine/tdengine:3.0.0.0 ``` diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 959438f48c288769186d22a9013fe10b738fd48f..eed8b9ca7dc0d0e6d8801b64071377062b523358 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -106,7 +106,7 @@ taos --dump-config | 适用范围 | 仅服务端适用 | | 含义 | 服务器内部的系统监控开关。监控主要负责收集物理节点的负载状况,包括 CPU、内存、硬盘、网络带宽的监控记录,监控信息将通过 HTTP 协议发送给由 `monitorFqdn` 和 `monitorProt` 指定的 TaosKeeper 监控服务 | | 取值范围 | 0:关闭监控服务, 1:激活监控服务。 | -| 缺省值 | 1 | +| 缺省值 | 0 | ### monitorFqdn @@ -136,12 +136,21 @@ taos --dump-config ### telemetryReporting -| 属性 | 说明 | -| -------- | ---------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 是否允许 TDengine 采集和上报基本使用信息 | -| 取值范围 | 0:不允许 1:允许 | -| 缺省值 | 1 | +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 telemetry | +| 取值范围 | 0,1 0: 不上传;1:上传 | +| 缺省值 | 1 | + +### crashReporting + +| 属性 | 说明 | +| -------- | -------------------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 |是否上传 crash 信息 | +| 取值范围 | 0,1 0: 不上传;1:上传 | +| 缺省值 | 1 | ## 查询相关 @@ -153,11 +162,7 @@ taos --dump-config | 含义 | 查询语句的执行策略 | | 单位 | 无 | | 缺省值 | 1 | -| 补充说明 | 1: 只使用 vnode,不使用 qnode | - -2: 没有扫描算子的子任务在 qnode 执行,带扫描算子的子任务在 vnode 执行 - -3: vnode 只运行扫描算子,其余算子均在 qnode 执行 | +| 补充说明 | 1: 只使用 vnode,不使用 qnode; 2: 没有扫描算子的子任务在 qnode 执行,带扫描算子的子任务在 vnode 执行; 3: vnode 只运行扫描算子,其余算子均在 qnode 执行 | ### querySmaOptimize @@ -167,11 +172,7 @@ taos --dump-config | 含义 | sma index 的优化策略 | | 单位 | 无 | | 缺省值 | 0 | -| 补充说明 | - -0: 表示不使用 sma index,永远从原始数据进行查询 - -1: 表示使用 sma index,对符合的语句,直接从预计算的结果进行查询 | +| 补充说明 |0: 表示不使用 sma index,永远从原始数据进行查询; 1: 表示使用 sma index,对符合的语句,直接从预计算的结果进行查询 | ### maxNumOfDistinctRes @@ -191,6 +192,16 @@ taos --dump-config | 取值范围 | 0 表示包含函数名,1 表示不包含函数名。 | | 缺省值 | 0 | +### countAlwaysReturnValue + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 仅服务端适用 | +| 含义 | count/hyperloglog函数在输入数据为空或者NULL的情况下是否返回值 | +| 取值范围 | 0:返回空行,1:返回 0 | +| 缺省值 | 1 | +| 补充说明 | 该参数设置为 1 时,如果查询中含有 GROUP BY,PARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL, 对应的组或窗口将不返回查询结果 | + ## 区域相关 ### timezone @@ -305,13 +316,22 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 数据文件目录,所有的数据文件都将写入该目录 | | 缺省值 | /var/lib/taos | +| 补充说明 | [多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8) 功能需要与 [KEEP](https://docs.taosdata.com/taos-sql/database/#%E5%8F%82%E6%95%B0%E8%AF%B4%E6%98%8E) 参数配合使用 | + +### tempDir + +| 属性 | 说明 | +| -------- | ------------------------------------------ | +| 适用范围 | 仅服务端适用 | +| 含义 | 该参数指定所有系统运行过程中的临时文件生成的目录 | +| 缺省值 | /tmp | ### minimalTmpDirGB | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 服务端和客户端均适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时,停止写临时文件 | +| 含义 | tempDir 所指定的临时文件目录所需要保留的最小空间 | | 单位 | GB | | 缺省值 | 1.0 | @@ -320,7 +340,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------------------------------ | | 适用范围 | 仅服务端适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时,停止写时序数据 | +| 含义 | dataDir 指定的时序数据存储目录所需要保留的最小 | | 单位 | GB | | 缺省值 | 2.0 | @@ -335,27 +355,7 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0-4096 | | 缺省值 | CPU 核数的 2 倍 | -## 时间相关 - -### statusInterval - -| 属性 | 说明 | -| -------- | --------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | dnode 向 mnode 报告状态间隔 | -| 单位 | 秒 | -| 取值范围 | 1-10 | -| 缺省值 | 1 | - -### shellActivityTimer - -| 属性 | 说明 | -| -------- | --------------------------------- | -| 适用范围 | 服务端和客户端均适用 | -| 含义 | shell 客户端向 mnode 发送心跳间隔 | -| 单位 | 秒 | -| 取值范围 | 1-120 | -| 缺省值 | 3 | +## 时间相关 | ## 性能调优 @@ -367,28 +367,6 @@ charset 的有效值是 UTF-8。 | 含义 | 设置写入线程的最大数量 | | 缺省值 | | -## 压缩相关 - -### compressMsgSize - -| 属性 | 说明 | -| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中,对通讯的消息进行压缩的阈值。如果要压缩消息,建议设置为 64330 字节,即大于 64330 字节的消息体才进行压缩。 | -| 单位 | bytes | -| 取值范围 | `0 `表示对所有的消息均进行压缩 >0: 超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - -### compressColData - -| 属性 | 说明 | -| -------- | --------------------------------------------------------------------------------------- | -| 适用范围 | 仅服务端适用 | -| 含义 | 客户端与服务器之间进行消息通讯过程中,对服务器端查询结果进行列压缩的阈值。 | -| 单位 | bytes | -| 取值范围 | 0: 对所有查询结果均进行压缩 >0: 查询结果中任意列大小超过该值的消息才进行压缩 -1: 不压缩 | -| 缺省值 | -1 | - ## 日志相关 ### logDir @@ -404,7 +382,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | -------------------------------------------- | | 适用范围 | 服务端和客户端均适用 | -| 含义 | 当日志文件夹的磁盘大小小于该值时,停止写日志 | +| 含义 | 当日志文件夹所在磁盘可用空间大小小于该值时,停止写日志 | | 单位 | GB | | 缺省值 | 1.0 | @@ -613,7 +591,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | schemaless 自定义的子表名 | +| 含义 | schemaless 自定义的子表名的 key | | 类型 | 字符串 | | 缺省值 | 无 | @@ -631,7 +609,7 @@ charset 的有效值是 UTF-8。 | 属性 | 说明 | | -------- | ----------------------------- | | 适用范围 | 仅客户端适用 | -| 含义 | schemaless 列数据是否顺序一致 | +| 含义 | schemaless 列数据是否顺序一致,从3.0.3.0开始,该配置废弃 | | 值域 | 0:不一致;1: 一致 | | 缺省值 | 1 | @@ -656,12 +634,18 @@ charset 的有效值是 UTF-8。 | 取值范围 | 0: 不启动;1:启动 | | 缺省值 | 1 | -## 2.X 与 3.0 配置参数对比 +## 压缩参数 -:::note -对于 2.x 版本中适用但在 3.0 版本中废弃的参数,其当前行为会有特别说明 +### compressMsgSize -::: +| 属性 | 说明 | +| -------- | ----------------------------- | +| 适用于 | 服务端和客户端均适用 | +| 含义 | 是否对 RPC 消息进行压缩 | +| 取值范围 | -1: 所有消息都不压缩; 0: 所有消息都压缩; N (N>0): 只有大于 N 个字节的消息才压缩 | +| 缺省值 | -1 | + +## 3.0 中有效的配置参数列表 | # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | | --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | @@ -674,159 +658,134 @@ charset 的有效值是 UTF-8。 | 7 | monitorFqdn | 否 | 是 | | | 8 | monitorPort | 否 | 是 | | | 9 | monitorInterval | 是 | 是 | | -| 10 | monitorMaxLogs | 否 | 是 | | -| 11 | monitorComp | 否 | 是 | | -| 12 | telemetryReporting | 是 | 是 | | -| 13 | telemetryInterval | 否 | 是 | | -| 14 | telemetryServer | 否 | 是 | | -| 15 | telemetryPort | 否 | 是 | | -| 16 | queryPolicy | 否 | 是 | | -| 17 | querySmaOptimize | 否 | 是 | | -| 18 | queryRsmaTolerance | 否 | 是 | | -| 19 | queryBufferSize | 是 | 是 | | -| 20 | maxNumOfDistinctRes | 是 | 是 | | -| 21 | minSlidingTime | 是 | 是 | | -| 22 | minIntervalTime | 是 | 是 | | -| 23 | countAlwaysReturnValue | 是 | 是 | | -| 24 | dataDir | 是 | 是 | | -| 25 | minimalDataDirGB | 是 | 是 | | -| 26 | supportVnodes | 否 | 是 | | -| 27 | tempDir | 是 | 是 | | -| 28 | minimalTmpDirGB | 是 | 是 | | -| 29 | compressMsgSize | 是 | 是 | | -| 30 | compressColData | 是 | 是 | | -| 31 | smlChildTableName | 是 | 是 | | -| 32 | smlTagName | 是 | 是 | | -| 33 | smlDataFormat | 否 | 是 | | -| 34 | statusInterval | 是 | 是 | | -| 35 | shellActivityTimer | 是 | 是 | | -| 36 | transPullupInterval | 否 | 是 | | -| 37 | mqRebalanceInterval | 否 | 是 | | -| 38 | ttlUnit | 否 | 是 | | -| 39 | ttlPushInterval | 否 | 是 | | -| 40 | numOfTaskQueueThreads | 否 | 是 | | -| 41 | numOfRpcThreads | 否 | 是 | | -| 42 | numOfCommitThreads | 是 | 是 | | -| 43 | numOfMnodeReadThreads | 否 | 是 | | -| 44 | numOfVnodeQueryThreads | 否 | 是 | | -| 45 | ratioOfVnodeStreamThreads | 否 | 是 | | -| 46 | numOfVnodeFetchThreads | 否 | 是 | | -| 47 | numOfVnodeRsmaThreads | 否 | 是 | | -| 48 | numOfQnodeQueryThreads | 否 | 是 | | -| 49 | numOfQnodeFetchThreads | 否 | 是 | | -| 50 | numOfSnodeSharedThreads | 否 | 是 | | -| 51 | numOfSnodeUniqueThreads | 否 | 是 | | -| 52 | rpcQueueMemoryAllowed | 否 | 是 | | -| 53 | logDir | 是 | 是 | | -| 54 | minimalLogDirGB | 是 | 是 | | -| 55 | numOfLogLines | 是 | 是 | | -| 56 | asyncLog | 是 | 是 | | -| 57 | logKeepDays | 是 | 是 | | -| 60 | debugFlag | 是 | 是 | | -| 61 | tmrDebugFlag | 是 | 是 | | -| 62 | uDebugFlag | 是 | 是 | | -| 63 | rpcDebugFlag | 是 | 是 | | -| 64 | jniDebugFlag | 是 | 是 | | -| 65 | qDebugFlag | 是 | 是 | | -| 66 | cDebugFlag | 是 | 是 | | -| 67 | dDebugFlag | 是 | 是 | | -| 68 | vDebugFlag | 是 | 是 | | -| 69 | mDebugFlag | 是 | 是 | | -| 70 | wDebugFlag | 是 | 是 | | -| 71 | sDebugFlag | 是 | 是 | | -| 72 | tsdbDebugFlag | 是 | 是 | | -| 73 | tqDebugFlag | 否 | 是 | | -| 74 | fsDebugFlag | 是 | 是 | | -| 75 | udfDebugFlag | 否 | 是 | | -| 76 | smaDebugFlag | 否 | 是 | | -| 77 | idxDebugFlag | 否 | 是 | | -| 78 | tdbDebugFlag | 否 | 是 | | -| 79 | metaDebugFlag | 否 | 是 | | -| 80 | timezone | 是 | 是 | | -| 81 | locale | 是 | 是 | | -| 82 | charset | 是 | 是 | | -| 83 | udf | 是 | 是 | | -| 84 | enableCoreFile | 是 | 是 | | -| 85 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | -| 86 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | -| 87 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | -| 88 | vnodeBak | 是 | 否 | 3.0 行为未知 | -| 89 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 | -| 90 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | -| 91 | offlineThreshold | 是 | 否 | 3.0 行为未知 | -| 92 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | -| 93 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | -| 94 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | -| 95 | rpcTimer | 是 | 否 | 3.0 行为未知 | -| 96 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | -| 97 | rpcForceTcp | 是 | 否 | 默认为 TCP | -| 98 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | -| 99 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | -| 100 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | -| 101 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | -| 102 | smlTagNullName | 是 | 否 | 3.0 行为未知 | -| 103 | keepColumnName | 是 | 否 | 3.0 行为未知 | -| 104 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | -| 105 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 106 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 107 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | -| 108 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | -| 109 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | -| 110 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 111 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 112 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | -| 113 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 114 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | -| 115 | days | 是 | 否 | 由 create db 的参数 duration 取代 | -| 116 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | -| 117 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | -| 118 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | -| 119 | quorum | 是 | 否 | 由 RAFT 协议决定 | -| 120 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | -| 121 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | -| 122 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | -| 123 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | -| 124 | partitions | 是 | 否 | 3.0 行为未知 | -| 125 | update | 是 | 否 | 允许更新部分列 | -| 126 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | -| 127 | maxSQLLength | 是 | 否 | SQL 上限为 1MB,无需参数控制 | -| 128 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | -| 129 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | -| 130 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | -| 131 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力,详见后面的 Note | -| 132 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | -| 133 | http | 是 | 否 | http 服务由 taosAdapter 提供 | -| 134 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | -| 135 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | -| 136 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | -| 137 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | -| 138 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | -| 139 | enableRecordSql | 是 | 否 | 3.0 行为未知 | -| 140 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | -| 141 | stream | 是 | 否 | 默认启用连续查询 | -| 142 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | -| 143 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | -| 144 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | -| 145 | walFlushSize | 是 | 否 | 3.0 行为未知 | -| 146 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | -| 147 | flowctrl | 是 | 否 | 3.0 行为未知 | -| 148 | slaveQuery | 是 | 否 | 3.0 行为未知: slave vnode 是否能够处理查询? | -| 149 | adjustMaster | 是 | 否 | 3.0 行为未知 | -| 150 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | -| 151 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | -| 152 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | -| 153 | clientMerge | 是 | 否 | 3.0 行为未知 | -| 154 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 155 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 156 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 157 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 158 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 159 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | -| 160 | probeSeconds | 是 | 否 | 3.0 行为未知 | -| 161 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | -| 162 | probeInterval | 是 | 否 | 3.0 行为未知 | -| 163 | lossyColumns | 是 | 否 | 3.0 行为未知 | -| 164 | fPrecision | 是 | 否 | 3.0 行为未知 | -| 165 | dPrecision | 是 | 否 | 3.0 行为未知 | -| 166 | maxRange | 是 | 否 | 3.0 行为未知 | -| 167 | range | 是 | 否 | 3.0 行为未知 | +| 10 | queryPolicy | 否 | 是 | | +| 11 | querySmaOptimize | 否 | 是 | | +| 12 | maxNumOfDistinctRes | 是 | 是 | | +| 15 | countAlwaysReturnValue | 是 | 是 | | +| 16 | dataDir | 是 | 是 | | +| 17 | minimalDataDirGB | 是 | 是 | | +| 18 | supportVnodes | 否 | 是 | | +| 19 | tempDir | 是 | 是 | | +| 20 | minimalTmpDirGB | 是 | 是 | | +| 21 | smlChildTableName | 是 | 是 | | +| 22 | smlTagName | 是 | 是 | | +| 23 | smlDataFormat | 否 | 是(从3.0.3.0开始,该配置废弃) | | +| 24 | statusInterval | 是 | 是 | | +| 25 | logDir | 是 | 是 | | +| 26 | minimalLogDirGB | 是 | 是 | | +| 27 | numOfLogLines | 是 | 是 | | +| 28 | asyncLog | 是 | 是 | | +| 29 | logKeepDays | 是 | 是 | | +| 30 | debugFlag | 是 | 是 | | +| 31 | tmrDebugFlag | 是 | 是 | | +| 32 | uDebugFlag | 是 | 是 | | +| 33 | rpcDebugFlag | 是 | 是 | | +| 34 | jniDebugFlag | 是 | 是 | | +| 35 | qDebugFlag | 是 | 是 | | +| 36 | cDebugFlag | 是 | 是 | | +| 37 | dDebugFlag | 是 | 是 | | +| 38 | vDebugFlag | 是 | 是 | | +| 39 | mDebugFlag | 是 | 是 | | +| 40 | wDebugFlag | 是 | 是 | | +| 41 | sDebugFlag | 是 | 是 | | +| 42 | tsdbDebugFlag | 是 | 是 | | +| 43 | tqDebugFlag | 否 | 是 | | +| 44 | fsDebugFlag | 是 | 是 | | +| 45 | udfDebugFlag | 否 | 是 | | +| 46 | smaDebugFlag | 否 | 是 | | +| 47 | idxDebugFlag | 否 | 是 | | +| 48 | tdbDebugFlag | 否 | 是 | | +| 49 | metaDebugFlag | 否 | 是 | | +| 50 | timezone | 是 | 是 | | +| 51 | locale | 是 | 是 | | +| 52 | charset | 是 | 是 | | +| 53 | udf | 是 | 是 | | +| 54 | enableCoreFile | 是 | 是 | | + +## 2.x->3.0 的废弃参数 + +| # | **参数** | **适用于 2.X ** | **适用于 3.0 ** | 3.0 版本的当前行为 | +| --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | +| 1 | arbitrator | 是 | 否 | 通过 RAFT 协议选主 | +| 2 | numOfThreadsPerCore | 是 | 否 | 有其它参数设置多种线程池的大小 | +| 3 | numOfMnodes | 是 | 否 | 通过 create mnode 命令动态创建 mnode | +| 4 | vnodeBak | 是 | 否 | 3.0 行为未知 | +| 5 | balance | 是 | 否 | 负载均衡功能由 split/merge vgroups 实现 (暂不支持) | +| 6 | balanceInterval | 是 | 否 | 随着 balance 参数失效 | +| 7 | offlineThreshold | 是 | 否 | 3.0 行为未知 | +| 8 | role | 是 | 否 | 由 supportVnode 决定是否能够创建 | +| 9 | dnodeNopLoop | 是 | 否 | 2.6 文档中未找到此参数 | +| 10 | keepTimeOffset | 是 | 否 | 2.6 文档中未找到此参数 | +| 11 | rpcTimer | 是 | 否 | 3.0 行为未知 | +| 12 | rpcMaxTime | 是 | 否 | 3.0 行为未知 | +| 13 | rpcForceTcp | 是 | 否 | 默认为 TCP | +| 14 | tcpConnTimeout | 是 | 否 | 3.0 行为未知 | +| 15 | syncCheckInterval | 是 | 否 | 3.0 行为未知 | +| 16 | maxTmrCtrl | 是 | 否 | 3.0 行为未知 | +| 17 | monitorReplica | 是 | 否 | 由 RAFT 协议管理多副本 | +| 18 | smlTagNullName | 是 | 否 | 3.0 行为未知 | +| 19 | keepColumnName | 是 | 否 | 3.0 行为未知 | +| 20 | ratioOfQueryCores | 是 | 否 | 由 线程池 相关配置参数决定 | +| 21 | maxStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 22 | maxFirstStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 23 | retryStreamCompDelay | 是 | 否 | 3.0 行为未知 | +| 24 | streamCompDelayRatio | 是 | 否 | 3.0 行为未知 | +| 25 | maxVgroupsPerDb | 是 | 否 | 由 create db 的参数 vgroups 指定实际 vgroups 数量 | +| 26 | maxTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 27 | minTablesPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 28 | tableIncStepPerVnode | 是 | 否 | DB 中的所有表近似平均分配到各个 vgroup | +| 29 | cache | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 30 | blocks | 是 | 否 | 由 buffer 代替 cache\*blocks | +| 31 | days | 是 | 否 | 由 create db 的参数 duration 取代 | +| 32 | keep | 是 | 否 | 由 create db 的参数 keep 取代 | +| 33 | minRows | 是 | 否 | 由 create db 的参数 minRows 取代 | +| 34 | maxRows | 是 | 否 | 由 create db 的参数 maxRows 取代 | +| 35 | quorum | 是 | 否 | 由 RAFT 协议决定 | +| 36 | comp | 是 | 否 | 由 create db 的参数 comp 取代 | +| 37 | walLevel | 是 | 否 | 由 create db 的参数 wal_level 取代 | +| 38 | fsync | 是 | 否 | 由 create db 的参数 wal_fsync_period 取代 | +| 39 | replica | 是 | 否 | 由 create db 的参数 replica 取代 | +| 40 | partitions | 是 | 否 | 3.0 行为未知 | +| 41 | update | 是 | 否 | 允许更新部分列 | +| 42 | cachelast | 是 | 否 | 由 create db 的参数 cacheModel 取代 | +| 43 | maxSQLLength | 是 | 否 | SQL 上限为 1MB,无需参数控制 | +| 44 | maxWildCardsLength | 是 | 否 | 3.0 行为未知 | +| 45 | maxRegexStringLen | 是 | 否 | 3.0 行为未知 | +| 46 | maxNumOfOrderedRes | 是 | 否 | 3.0 行为未知 | +| 47 | maxConnections | 是 | 否 | 取决于系统配置和系统处理能力,详见后面的 Note | +| 48 | mnodeEqualVnodeNum | 是 | 否 | 3.0 行为未知 | +| 49 | http | 是 | 否 | http 服务由 taosAdapter 提供 | +| 50 | httpEnableRecordSql | 是 | 否 | taosd 不提供 http 服务 | +| 51 | httpMaxThreads | 是 | 否 | taosd 不提供 http 服务 | +| 52 | restfulRowLimit | 是 | 否 | taosd 不提供 http 服务 | +| 53 | httpDbNameMandatory | 是 | 否 | taosd 不提供 http 服务 | +| 54 | httpKeepAlive | 是 | 否 | taosd 不提供 http 服务 | +| 55 | enableRecordSql | 是 | 否 | 3.0 行为未知 | +| 56 | maxBinaryDisplayWidth | 是 | 否 | 3.0 行为未知 | +| 57 | stream | 是 | 否 | 默认启用连续查询 | +| 58 | retrieveBlockingModel | 是 | 否 | 3.0 行为未知 | +| 59 | tsdbMetaCompactRatio | 是 | 否 | 3.0 行为未知 | +| 60 | defaultJSONStrType | 是 | 否 | 3.0 行为未知 | +| 61 | walFlushSize | 是 | 否 | 3.0 行为未知 | +| 62 | keepTimeOffset | 是 | 否 | 3.0 行为未知 | +| 63 | flowctrl | 是 | 否 | 3.0 行为未知 | +| 64 | slaveQuery | 是 | 否 | 3.0 行为未知: slave vnode 是否能够处理查询? | +| 65 | adjustMaster | 是 | 否 | 3.0 行为未知 | +| 66 | topicBinaryLen | 是 | 否 | 3.0 行为未知 | +| 67 | telegrafUseFieldNum | 是 | 否 | 3.0 行为未知 | +| 68 | deadLockKillQuery | 是 | 否 | 3.0 行为未知 | +| 69 | clientMerge | 是 | 否 | 3.0 行为未知 | +| 70 | sdbDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 71 | odbcDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 72 | httpDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 73 | monDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 74 | cqDebugFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 75 | shortcutFlag | 是 | 否 | 参考 3.0 的 DebugFlag 系列参数 | +| 76 | probeSeconds | 是 | 否 | 3.0 行为未知 | +| 77 | probeKillSeconds | 是 | 否 | 3.0 行为未知 | +| 78 | probeInterval | 是 | 否 | 3.0 行为未知 | +| 79 | lossyColumns | 是 | 否 | 3.0 行为未知 | +| 80 | fPrecision | 是 | 否 | 3.0 行为未知 | +| 81 | dPrecision | 是 | 否 | 3.0 行为未知 | +| 82 | maxRange | 是 | 否 | 3.0 行为未知 | +| 83 | range | 是 | 否 | 3.0 行为未知 | diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index 3aebd616a075394dbeddc9e608cd2a57bb8cf844..3d0bac25d20953733d842e8057f2c0c24deb831e 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -83,7 +83,7 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 NULL。 6. 对 BINARY 或 NCHAR 列,如果数据行中所提供值的长度超出了列类型的限制,自动增加该列允许存储的字符长度上限(只增不减),以保证数据的完整保存。 7. 整个处理过程中遇到的错误会中断写入过程,并返回错误代码。 -8. 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field,后面的数据按照这个顺序),如果顺序不一样,需要配置参数 smlDataFormat 为 false,否则,数据写入按照相同顺序写入,库中数据会异常。 +8. 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field,后面的数据按照这个顺序),如果顺序不一样,需要配置参数 smlDataFormat 为 false,否则,数据写入按照相同顺序写入,库中数据会异常,从3.0.3.0开始,自动检测顺序是否一致,该配置废弃。 :::tip 无模式所有的处理逻辑,仍会遵循 TDengine 对数据结构的底层限制,例如每行数据的总长度不能超过 diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 7780dc2fe9a5742ce9d6855599ca080804a493b5..bbd5b1b9116ea0a8d2dab4c26c37d0d07cb17d78 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -24,7 +24,15 @@ taosKeeper 安装方式: taosKeeper 需要在操作系统终端执行,该工具支持三种配置方式:[命令行参数](#命令行参数启动)、[环境变量](#环境变量启动) 和 [配置文件](#配置文件启动)。优先级为:命令行参数、环境变量、配置文件参数。 -**在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** 并且 TDengine 已经开启监控服务,具体请参考:[TDengine 监控配置](../config/#监控相关)。 +**在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** 并且 TDengine 已经开启监控服务,TDengine 配置文件 `taos.cfg` 中至少需要配置 `monitor` 和 `monitorFqdn`。 + +```shell +monitor 1 +monitorFqdn localhost # taoskeeper 服务的 FQDN +``` + +TDengine 监控配置相关,具体请参考:[TDengine 监控配置](../config/#监控相关)。 + ### 命令行参数启动 diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx index d5cfe847cada26a102b634573e7f4e2492adf60b..5927dc4fcacc0b947e3a4c44778d9d5979bfc196 100644 --- a/docs/zh/20-third-party/01-grafana.mdx +++ b/docs/zh/20-third-party/01-grafana.mdx @@ -77,7 +77,7 @@ sudo -u grafana grafana-cli plugins install tdengine-datasource 或者从 [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) 或 [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) 下载 .zip 文件到本地并解压到 Grafana 插件目录。命令行下载示例如下: ```bash -GF_VERSION=3.2.7 +GF_VERSION=3.2.9 # from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip # from Grafana @@ -156,13 +156,13 @@ docker run -d \ services: tdengine: - image: tdengine/tdengine:2.6.0.2 + image: tdengine/tdengine:3.0.2.4 environment: TAOS_FQDN: tdengine volumes: - tdengine-data:/var/lib/taos/ grafana: - image: grafana/grafana:8.5.6 + image: grafana/grafana:9.3.6 volumes: - ./tdengine.yml/:/etc/grafana/provisioning/tdengine.yml - grafana-data:/var/lib/grafana @@ -197,11 +197,17 @@ docker run -d \ - INPUT SQL:输入要查询的语句(该 SQL 语句的结果集应为两列多行),例如:`select _wstart, avg(mem_system) from log.dnodes_info where ts >= $from and ts < $to interval($interval)` ,其中,from、to 和 interval 为 TDengine 插件的内置变量,表示从 Grafana 插件面板获取的查询范围和时间间隔。除了内置变量外,`也支持可以使用自定义模板变量`。 - ALIAS BY:可设置当前查询别名。 - GENERATE SQL: 点击该按钮会自动替换相应变量,并生成最终执行的语句。 +- Group by column name(s): **半角**逗号分隔的 `group by` 或 `partition by` 列名。如果是 `group by` or `partition by` 查询语句,设置 `Group by` 列,可以展示多维数据。例如:INPUT SQL 为 `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)`,设置 Group by 列名为 `dnode_ep`,可以按 `dnode_ep` 展示数据。 +- Format to: Group by 或 Partition by 场景下多维数据 legend 格式化格式。例如上述 INPUT SQL,将 Format to 设置为 `mem_system_{{dnode_ep}}`,展示的 legend 名字为格式化的列名。 按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下: ![TDengine Database Grafana plugin create dashboard](./create_dashboard2.webp) +查询每台 TDengine 服务器指定间隔系统内存平均使用量如下: + +![TDengine Database Grafana plugin create dashboard](./create_dashboard3.webp) + > 关于如何使用 Grafana 创建相应的监测界面以及更多有关使用 Grafana 的信息,请参考 Grafana 官方的[文档](https://grafana.com/docs/)。 ### 导入 Dashboard diff --git a/docs/zh/20-third-party/create_dashboard1.webp b/docs/zh/20-third-party/create_dashboard1.webp index 55eb388833e4df2a46f4d1cf6d346aa11429385d..df41d66d50e888555792ef34188a966c2e106a86 100644 Binary files a/docs/zh/20-third-party/create_dashboard1.webp and b/docs/zh/20-third-party/create_dashboard1.webp differ diff --git a/docs/zh/20-third-party/create_dashboard2.webp b/docs/zh/20-third-party/create_dashboard2.webp index bb40e407187718c52e9f617d8ebd3d25fd14b56b..ba50f5299605bd72ebfe1f3f5d1d62bdbc8939c9 100644 Binary files a/docs/zh/20-third-party/create_dashboard2.webp and b/docs/zh/20-third-party/create_dashboard2.webp differ diff --git a/docs/zh/20-third-party/create_dashboard3.webp b/docs/zh/20-third-party/create_dashboard3.webp new file mode 100644 index 0000000000000000000000000000000000000000..84f387f87bcc8e06fca90deabebd1d601c528ef1 Binary files /dev/null and b/docs/zh/20-third-party/create_dashboard3.webp differ diff --git a/docs/zh/25-application/_03-immigrate.md b/docs/zh/25-application/03-immigrate.md similarity index 70% rename from docs/zh/25-application/_03-immigrate.md rename to docs/zh/25-application/03-immigrate.md index d1c9caea099b79494784aa1122e89d7b4d412464..75788c0cc7d6e0e84402ba77c4a1aa875d772d8b 100644 --- a/docs/zh/25-application/_03-immigrate.md +++ b/docs/zh/25-application/03-immigrate.md @@ -18,82 +18,15 @@ title: OpenTSDB 应用迁移到 TDengine 的最佳实践 如果我们将原本运行在 OpenTSDB 上的应用迁移到 TDengine 上,不仅可以有效地降低计算和存储资源的占用、减少部署服务器的规模,还能够极大减少运行维护的成本的输出,让运维管理工作更简单、更轻松,大幅降低总拥有成本。与 OpenTSDB 一样,TDengine 也已经进行了开源,不同的是,除了单机版,后者还实现了集群版开源,被厂商绑定的顾虑一扫而空。 -在下文中我们将就“使用最典型并广泛应用的运维监控(DevOps)场景”来说明,如何在不编码的情况下将 OpenTSDB 的应用快速、安全、可靠地迁移到 TDengine 之上。后续的章节会做更深度的介绍,以便于进行非 DevOps 场景的迁移。 +在下文中我们将说明如何在不编码的情况下将 OpenTSDB 的应用快速、安全、可靠地迁移到 TDengine 之上。 -## DevOps 应用快速迁移 +## TDengine 与 OpenTSDB 的差异 -### 1、典型应用场景 - -一个典型的 DevOps 应用场景的系统整体的架构如下图(图 1) 所示。 - -**图 1. DevOps 场景中典型架构** -![TDengine Database IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](./IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.webp "图1. DevOps 场景中典型架构") - -在该应用场景中,包含了部署在应用环境中负责收集机器度量(Metrics)、网络度量(Metrics)以及应用度量(Metrics)的 Agent 工具、汇聚 Agent 收集信息的数据收集器,数据持久化存储和管理的系统以及监控数据可视化工具(例如:Grafana 等)。 - -其中,部署在应用节点的 Agents 负责向 collectd/Statsd 提供不同来源的运行指标,collectd/StatsD 则负责将汇聚的数据推送到 OpenTSDB 集群系统,然后使用可视化看板 Grafana 将数据可视化呈现出来。 - -### 2、迁移服务 - -- **TDengine 安装部署** - -首先是 TDengine 的安装,从官网上下载 TDengine 最新稳定版进行安装。各种安装包的使用帮助请参见博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 - -注意,安装完成以后,不要立即启动 `taosd` 服务,在正确配置完成参数以后再启动。 - -- **调整数据收集器配置** - -在 TDengine 2.4 版本中,包含一个组件 taosAdapter。taosAdapter 是一个无状态、可快速弹性伸缩的组件,它可以兼容 Influxdb 的 Line Protocol 和 OpenTSDB 的 telnet/JSON 写入协议规范,提供了丰富的数据接入能力,有效的节省用户迁移成本,降低用户应用迁移的难度。 - -用户可以根据需求弹性部署 taosAdapter 实例,结合场景的需要,快速提升数据写入的吞吐量,为不同应用场景下的数据写入提供保障。 - -通过 taosAdapter,用户可以将 collectd 或 StatsD 收集的数据直接推送到 TDengine ,实现应用场景的无缝迁移,非常的轻松便捷。taosAdapter 还支持 Telegraf、Icinga、TCollector 、node_exporter 的数据接入,使用详情参考[taosAdapter](/reference/taosadapter/)。 - -如果使用 collectd,修改其默认位置 `/etc/collectd/collectd.conf` 的配置文件为指向 taosAdapter 部署的节点 IP 地址和端口。假设 taosAdapter 的 IP 地址为 192.168.1.130,端口为 6046,配置如下: - -```html -LoadPlugin write_tsdb - - - Host "192.168.1.130" Port "6046" HostTags "status=production" StoreRates - false AlwaysAppendDS false - - -``` - -即可让 collectd 将数据使用推送到 OpenTSDB 的插件方式推送到 taosAdapter, taosAdapter 将调用 API 将数据写入到 TDengine 中,从而完成数据的写入工作。如果你使用的是 StatsD 相应地调整配置文件信息。 - -- **调整看板(Dashboard)系统** - -在数据能够正常写入 TDengine 后,可以调整适配 Grafana 将写入 TDengine 的数据可视化呈现出来。获取和使用 TDengine 提供的 Grafana 插件请参考[与其他工具的连接](/third-party/grafana)。 - -TDengine 提供了默认的两套 Dashboard 模板,用户只需要将 Grafana 目录下的模板导入到 Grafana 中即可激活使用。 - -**图 2. 导入 Grafana 模板** -![TDengine Database IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard](./IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.webp "图2. 导入 Grafana 模板") - -操作完以上步骤后,就完成了将 OpenTSDB 替换成为 TDengine 的迁移工作。可以看到整个流程非常简单,不需要写代码,只需要对某些配置文件进行调整即可完成全部的迁移工作。 - -### 3、迁移后架构 - -完成迁移以后,此时的系统整体的架构如下图(图 3)所示,而整个过程中采集端、数据写入端、以及监控呈现端均保持了稳定,除了极少的配置调整外,不涉及任何重要的更改和变动。OpenTSDB 大量的应用场景均为 DevOps ,这种场景下,简单的参数设置即可完成 OpenTSDB 到 TDengine 迁移动作,使用上 TDengine 更加强大的处理能力和查询性能。 - -在绝大多数的 DevOps 场景中,如果你拥有一个小规模的 OpenTSDB 集群(3 台及以下的节点)作为 DevOps 的存储端,依赖于 OpenTSDB 为系统持久化层提供数据存储和查询功能,那么你可以安全地将其替换为 TDengine,并节约更多的计算和存储资源。在同等计算资源配置情况下,单台 TDengine 即可满足 3 ~ 5 台 OpenTSDB 节点提供的服务能力。如果规模比较大,那便需要采用 TDengine 集群。 - -如果你的应用特别复杂,或者应用领域并不是 DevOps 场景,你可以继续阅读后续的章节,更加全面深入地了解将 OpenTSDB 的应用迁移到 TDengine 的高级话题。 - -**图 3. 迁移完成后的系统架构** -![TDengine Database IT-DevOps-Solutions-Immigrate-TDengine-Arch](./IT-DevOps-Solutions-Immigrate-TDengine-Arch.webp "图 3. 迁移完成后的系统架构") - -## 其他场景的迁移评估与策略 - -### 1、TDengine 与 OpenTSDB 的差异 - -本章将详细介绍 OpenTSDB 与 TDengine 在系统功能层面上存在的差异。阅读完本章的内容,你可以全面地评估是否能够将某些基于 OpenTSDB 的复杂应用迁移到 TDengine 上,以及迁移之后应该注意的问题。 +本节将详细介绍 OpenTSDB 与 TDengine 在系统功能层面上存在的差异。阅读完本节的内容,你可以全面地评估是否能够将某些基于 OpenTSDB 的复杂应用迁移到 TDengine 上,以及迁移之后应该注意的问题。 TDengine 当前只支持 Grafana 的可视化看板呈现,所以如果你的应用中使用了 Grafana 以外的前端看板(例如[TSDash](https://github.com/facebook/tsdash)、[Status Wolf](https://github.com/box/StatusWolf)等),那么前端看板将无法直接迁移到 TDengine,需要将前端看板重新适配到 Grafana 才可以正常运行。 -在 2.3.0.x 版本中,TDengine 只能够支持 collectd 和 StatsD 作为数据收集汇聚软件,当然后面会陆续提供更多的数据收集聚合软件的接入支持。如果您的收集端使用了其他类型的数据汇聚器,您的应用需要适配到这两个数据汇聚端系统,才能够将数据正常写入。除了上述两个数据汇聚端软件协议以外,TDengine 还支持通过 InfluxDB 的行协议和 OpenTSDB 的数据写入协议、JSON 格式将数据直接写入,您可以重写数据推送端的逻辑,使用 TDengine 支持的行协议来写入数据。 +如果您的收集端使用了像 collectd 和 StatsD 这样的数据采集工具,要重新配置这些数据采集工具将数据写入到 TDengine。TDengine 还支持通过 InfluxDB 的行协议和 OpenTSDB 的数据写入协议、JSON 格式将数据直接写入,您可以重写数据推送端的逻辑,使用 TDengine 支持的行协议来写入数据。 此外,如果你的应用中使用了 OpenTSDB 以下特性,在将应用迁移到 TDengine 之前你还需要了解以下注意事项: @@ -104,11 +37,11 @@ TDengine 当前只支持 Grafana 的可视化看板呈现,所以如果你的 通过上面的介绍,相信你应该能够了解 OpenTSDB 迁移到 TDengine 带来的变化,这些信息也有助于你正确地判断是否可以接受将应用 迁移到 TDengine 之上,体验 TDengine 提供的强大的时序数据处理能力和便捷的使用体验。 -### 2、迁移策略 +## 迁移策略 首先将基于 OpenTSDB 的系统进行迁移涉及到的数据模式设计、系统规模估算、数据写入端改造,进行数据分流、应用适配工作;之后将两个系统并行运行一段时间,再将历史数据迁移到 TDengine 中。当然如果你的应用中有部分功能强依赖于上述 OpenTSDB 特性,同时又不希望停止使用,可以考虑保持原有的 OpenTSDB 系统运行,同时启动 TDengine 来提供主要的服务。 -## 数据模型设计 +### 数据模型设计 一方面,TDengine 要求其入库的数据具有严格的模式定义。另一方面,TDengine 的数据模型相对于 OpenTSDB 来说又更加丰富,多值模型能够兼容全部的单值模型的建立需求。 @@ -150,7 +83,7 @@ insert into memory_vm130_memory_buffered_collectd using memory tags(‘vm130’ 如果你想要利用 TDengine 的多值模型能力,需要首先满足以下要求:不同的采集量具有相同的采集频率,且能够通过消息队列**同时到达**数据写入端,从而确保使用 SQL 语句将多个指标一次性写入。将度量的名称作为超级表的名称,建立具有相同采集频率且能够同时到达的数据多列模型。子表的表名采用具有固定规则的方式进行命名。上述每个度量均只包含一个测量值,因此无法将其转化为多值模型。 -## 数据分流与应用适配 +### 数据分流与应用适配 从消息队列中订阅数据,并启动调整后的写入程序写入数据。 @@ -166,15 +99,128 @@ TDengine 不支持采用 OpenTSDB 的查询语法进行查询或数据获取处 TDengine 支持标准的 JDBC 3.0 接口操纵数据库,你也可以使用其他类型的高级语言的连接器来查询读取数据,以适配你的应用。具体的操作和使用帮助也请参阅用户手册。 -## 历史数据迁移 - -### 1、使用工具自动迁移数据 - -为了方便历史数据的迁移工作,我们为数据同步工具 DataX 提供了插件,能够将数据自动写入到 TDengine 中,需要注意的是 DataX 的自动化数据迁移只能够支持单值模型的数据迁移过程。 - -DataX 具体的使用方式及如何使用 DataX 将数据写入 TDengine 请参见[基于 DataX 的 TDengine 数据迁移工具](https://www.taosdata.com/blog/2021/10/26/3156.html)。 -在对 DataX 进行迁移实践后,我们发现通过启动多个进程,同时迁移多个 metric 的方式,可以大幅度的提高迁移历史数据的效率,下面是迁移过程中的部分记录,希望这些能为应用迁移工作带来参考。 +## 使用 DataX 迁移数据 + +为了方便历史数据的迁移工作,我们为数据同步工具 DataX 提供了适配 TDengine 3.0 的插件,能够将数据自动写入到 TDengine 中,需要注意的是 DataX 的自动化数据迁移只能够支持单值模型的数据迁移过程。 + +### 安装和部署 TDengine + +在进行数据迁移之前,要有一个正确运行的 TDengine 集群。首先是 TDengine 的安装,从官网上下载 TDengine 最新稳定版进行安装。各种安装包的使用帮助请参考 [安装指南](../../get-started/package) + +安装完成后,请根据 [部署指南](../../deployment/deploy) 配置集群。 + +### 插件功能介绍 + +1. TDengine30Reader 提供的功能: + 1. 支持通过 SQL 进行数据筛选; + 2. 根据时间间隔进行任务切分; + 3. 支持 TDengine 的全部数据类型; + 4. 支持批量读取,通过 batchSize 参数控制批量拉取结果集的大小,提高读取性能。 +2. TDengine30Writer 支持的功能: + 1. 支持 OpenTSDB 的 json 格式的行协议,使用 TDengine 的 schemaless 方式写入 TDengine。 + 2. 支持批量写入,通过 batchSize 参数控制批量写入的数量,提高写入性能。 + +### DataX 安装环境准备 + +1. 需要安装 TDengine 客户端 +2. 需要安装 JDK 1.8 环境(运行 DataX) +3. 需要安装 Python 环境(运行 DataX) +4. 需要 maven 编译环境(如果不编译 DataX 则可以不安装 maven) + +### 安装 + +1. 下载源码 +~~~ +git clone https://github.com/taosdata/DataX.git +~~~ +2. 编译打包 +~~~ +cd DataX +mvn -U clean package assembly:assembly -Dmaven.test.skip=true +~~~ +3. 安装 +~~~ +cp target/datax.tar.gz your_install_dir +cd your_install_dir +tar -zxvf dataX.tar.gz +~~~ + +### 数据迁移 Job 的配置 + +以一个从 OpenTSDB 到 TDengine 3.0 版本的数据迁移任务为例,配置文件 opentsdb2tdengine.json 如下: +~~~ +{ + "job":{ + "content":[{ + "reader": { + "name": "opentsdbreader", + "parameter": { + "endpoint": "http://192.168.1.180:4242", + "column": ["weather_temperature"], + "beginDateTime": "2021-01-01 00:00:00", + "endDateTime": "2021-01-01 01:00:00" + } + }, + "writer": { + "name": "tdengine30writer", + "parameter": { + "username": "root", + "password": "taosdata", + "connection": [ + { + "table": [ + "matric1" + ], + "jdbcUrl": "jdbc:TAOS://192.168.1.101:6030/test?timestampFormat=TIMESTAMP" + } + ], + "batchSize": 1000, + "ignoreTagsUnmatched": true + } + } + }], + "setting": { + "speed": { + "channel": 1 + } + } + } + } +~~~ +配置说明: +1. 上面的配置表示,从 192.168.1.180 的 OpenTSDB,到 192.168.1.101 的 TDengine 的迁移。迁移 metric 为 weather_temperature,时间从 2021-01-01 00:00:00 开始,到 2021-01-01 01:00:00 结束的数据。 +2. reader 使用 datax 的 opentsdbreader,parameter 的配置请参考:[opentsdbreader.md#配置参数](https://github.com/taosdata/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md) +3. tdengine30writer 的 parameter 中,user,password 为必须项,没有默认值。batchSize 不是必须项,默认值为 1。详细参考:[tdengine30writer.md#配置参数](https://github.com/taosdata/DataX/blob/master/tdengine30writer/doc/tdengine30writer-CN.md) +4. TDengine 中,如果 dbname 指定的 database 不存在,则需要在迁移前创建数据库。 + +### 执行迁移任务 + +~~~ +python bin/datax.py job/opentsdb2tdengine.json +~~~ + +### 限制条件 + +1. 目前,DataX 自带的 opentsdbreader 仅支持 OpenTSDB-2.3.X 版本。详细参考:[opentsdbreader#约束限制](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md#5-%E7%BA%A6%E6%9D%9F%E9%99%90%E5%88%B6) +2. 数据迁移工具依赖 TDengine 客户端中的 `libtaos.so/taos.dll/libtaos.dylib`,需要与服务端对应版本的 TDengine-client。 + +### FAQ + +1. 如何估算一个数据迁移任务所需要的资源 + DataX 的每个 reader 按照自己的 task 切分策略进行任务划分,具体请参考 DataX 的任务调度规则。在估算资源是,需要按照数据迁移的数据量,任务切分规则和网络带宽限制等综合考虑,最好以实际数据迁移测试结果为准。 +2. TDengine30Writer 的 batchSize 设置多大效率最高? + batchSize 是控制批量写入的参数,在获取 batchSize 行纪录后,TDengineWriter 会向 TDengine 发送一次写入请求,这减少了与 TDengine 交互次数,从而提高了性能。从测试结果来看,batchSize 在 500-1000 范围内效率最高。 +3. job 的配置中 channel 数为多少合适? + job 中的 channel 数为流量控制的参数,每个 channel 都需要开辟一块内存,用来缓存数据。如果 channel 设置过大,会引起 OOM,所以 channel 数并不是越大越好。增加 channel 数后,需要提高 JVM 内存大小。从测试结果来看,channel 在 1~6 的范围内都是合适,能够保证 DataX 的流量最大化即可。 +4. java.sql.SQLException: TDengine ERROR (8000060b): Invalid client value + 配置文件中 column 中没有配置 tbname,此时会触发行协议数据写入(行协议写入只会自动创建子表名,但需要提前创建好超级表),行协议写入的情况下不支持 TAG 数据类型为非 NCHAR,所以此种情况有两种解决方案:1.将 TAG 全部修改为 NCHAR 类型;2.在 Column 中配置好表名称这样不会触发行协议写入。 +5. java.sql.SQLException: TDengine ERROR (8000060b): Timestamp data out of range + 配置文件中 column 中没有配置 tbname,此时会触发行协议数据写入,且 TAG 全部为 NCHAR 类型,此时需要保证时间戳的一列名称为 _ts,而不能是其他名称(行协议写入下,默认将最后的时间戳写入到 _ts 一列,且不能随意命名)。若想避免请使用 tbname 指定表名以避免触发行协议写入。 + +### 提升性能 + + 在对 DataX 进行迁移实践后,我们发现通过启动多个进程,同时迁移多个 metric 的方式,可以大幅度的提高迁移历史数据的效率,下面是迁移过程中的部分记录,希望这些能为应用迁移工作带来参考。 | DataX 实例个数 (并发进程个数) | 迁移记录速度 (条/秒) | | ----------------------------- | --------------------- | @@ -184,9 +230,9 @@ DataX 具体的使用方式及如何使用 DataX 将数据写入 TDengine 请参 | 5 | 约 29.5 万 | | 10 | 约 33 万 | -
(注:测试数据源自 单节点 Intel(R) Core(TM) i7-10700 CPU@2.90GHz 16 核 64G 硬件设备,channel 和 batchSize 分别为 8 和 1000,每条记录包含 10 个 tag) +(注:测试数据源自 单节点 Intel(R) Core(TM) i7-10700 CPU@2.90GHz 16 核 64G 硬件设备,channel 和 batchSize 分别为 8 和 1000,每条记录包含 10 个 tag) -### 2、手动迁移数据 +## 手动迁移数据 如果你需要使用多值模型进行数据写入,就需要自行开发一个将数据从 OpenTSDB 导出的工具,然后确认哪些时间线能够合并导入到同一个时间线,再将可以同时导入的时间通过 SQL 语句的写入到数据库中。 @@ -393,31 +439,11 @@ WHERE ts>=1510560000 AND ts<=1515000009 综上所述,可使用单台 16 核 32GB 的机器,或者使用 2 台 8 核 16GB 机器构成的集群。 -## 附录 3: 集群部署及启动 - -TDengine 提供了丰富的帮助文档说明集群安装、部署的诸多方面的内容,这里提供相应的文档列表,供你参考。 - -### 集群部署 - -首先是安装 TDengine,从官网上下载 TDengine 最新稳定版,解压缩后运行 install.sh 进行安装。各种安装包的使用帮助请参见博客[《TDengine 多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html)。 - -注意安装完成以后,不要立即启动 `taosd` 服务,在正确配置完成参数以后才启动 `taosd` 服务。 - -### 设置运行参数并启动服务 - -为确保系统能够正常获取运行的必要信息。请在服务端正确设置以下关键参数: - -FQDN、firstEp、secondEP、dataDir、logDir、tmpDir、serverPort。各参数的具体含义及设置的要求,可参见文档《[TDengine 集群安装、管理](/cluster/)》 - -按照相同的步骤,在需要运行的节点上设置参数,并启动 `taosd` 服务,然后添加 Dnode 到集群中。 - -最后启动 `taos` 命令行程序,执行命令 `show dnodes`,如果能看到所有的加入集群的节点,那么集群顺利搭建完成。具体的操作流程及注意事项,请参阅文档《[TDengine 集群安装、管理](/cluster/)》 - -## 附录 4: 超级表名称 +## 附录 3: 超级表名称 由于 OpenTSDB 的 metric 名称中带有点号(“.”),例如“cpu.usage_user”这种名称的 metric。但是点号在 TDengine 中具有特殊含义,是用来分隔数据库和表名称的分隔符。TDengine 也提供转义符,以允许用户在(超级)表名称中使用关键词或特殊分隔符(如:点号)。为了使用特殊字符,需要采用转义字符将表的名称括起来,例如:`cpu.usage_user`这样就是合法的(超级)表名称。 -## 附录 5:参考文章 +## 附录 4:参考文章 -1. [使用 TDengine + collectd/StatsD + Grafana 快速搭建 IT 运维监控系统](/application/collectd/) -2. [通过 collectd 将采集数据直接写入 TDengine](/third-party/collectd/) +1. [使用 TDengine + collectd/StatsD + Grafana 快速搭建 IT 运维监控系统](../collectd/) +2. [通过 collectd 将采集数据直接写入 TDengine](../../third-party/collectd/) diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 595b69b08b87ee33e27937fb89b84adc41c89d08..b0006196301c8a5dfa06842f9f49f77ece7f7507 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -243,3 +243,8 @@ sudo launchctl load -w /Library/LaunchDaemons/limit.maxfiles.plist ``` launchctl limit maxfiles ``` +### 20 建库时提示 Out of dnodes +该提示是创建 db 的 vnode 数量不够了,需要的 vnode 不能超过了 dnode 中 vnode 的上限。因为系统默认是一个 dnode 中有 CPU 核数两倍的 vnode,也可以通过配置文件中的参数 supportVnodes 控制。 +正常调大 taos.cfg 中 supportVnodes 参数即可。 + + diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 7ed9e0c5a018401e2028a1e0786459d4e26f27b6..c9505d95a58ef03d5d374d952c4b66c3d5e37828 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,11 +10,38 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.2.6 + + + +## 3.0.2.5 + + + +## 3.0.2.4 + + + +## 3.0.2.3 + + + +## 3.0.2.2 + + + +## 3.0.2.1 + + + +## 3.0.2.0 + + + ## 3.0.1.8 - ## 3.0.1.7 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 67ca3fae67b36e5f08c57440bbaa64ec4f80bf4e..69d35f95c85c743b7f0f0ff035c23ed96c26b021 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,34 @@ taosTools 各版本安装包下载链接如下: import Release from "/components/ReleaseV3"; +## 2.4.6 + + + +## 2.4.3 + + + +## 2.4.2 + + + +## 2.4.1 + + + +## 2.4.0 + + + +## 2.3.3 + + + +## 2.3.2 + + + ## 2.3.0 diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index c86cd44354d5b8a4e59e8c12fde32b4c06dc376a..91ec6f24b190fdc1c4a4c7f9eba479dc2fc1125d 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -92,7 +92,7 @@ int main(int argc, char *argv[]) } // a simple way to parse input parameters - if (argc >= 3) strcpy(db, argv[2]); + if (argc >= 3) strncpy(db, argv[2], sizeof(db) - 1); if (argc >= 4) points = atoi(argv[3]); if (argc >= 5) numOfTables = atoi(argv[4]); diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 525f3e106ffb9d57f1fe915c838f925c1b7a0119..83b40c4f20a332b94abe45dd5c4fb7afd820861a 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -189,46 +189,54 @@ void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { tmq_t* build_consumer() { tmq_conf_res_t code; - tmq_t* tmq = NULL; + tmq_t* tmq = NULL; - tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_t* conf = tmq_conf_new(); code = tmq_conf_set(conf, "enable.auto.commit", "true"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "group.id", "cgrpName"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "client.id", "user defined name"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "td.connect.user", "root"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } code = tmq_conf_set(conf, "experimental.snapshot.enable", "false"); if (TMQ_CONF_OK != code) { - goto _end; + tmq_conf_destroy(conf); + return NULL; } tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq = tmq_consumer_new(conf, NULL, 0); - _end: +_end: tmq_conf_destroy(conf); return tmq; } diff --git a/examples/nodejs/node-example-raw.js b/examples/nodejs/node-example-raw.js deleted file mode 100644 index 058a50c4c3f0f7d44da4bdcc4fc54b81d7e1bc38..0000000000000000000000000000000000000000 --- a/examples/nodejs/node-example-raw.js +++ /dev/null @@ -1,135 +0,0 @@ -/* This example is to show how to use the td-connector through the cursor only and is a bit more raw. - * No promises, object wrappers around data, functions that prettify the data, or anything. - * The cursor will generally use callback functions over promises, and return and store the raw data from the C Interface. - * It is advised to use the td-connector through the cursor and the TaosQuery class amongst other higher level APIs. -*/ - -// Get the td-connector package -const taos = require('td2.0-connector'); - -/* We will connect to TDengine by passing an object comprised of connection options to taos.connect and store the - * connection to the variable conn - */ -/* - * Connection Options - * host: the host to connect to - * user: the use to login as - * password: the password for the above user to login - * config: the location of the taos.cfg file, by default it is in /etc/taos - * port: the port we connect through - */ -var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0}); - -// Initialize our TDengineCursor, which we use to interact with TDengine -var c1 = conn.cursor(); - -// c1.execute(query) will execute the query -// Let's create a database named db -try { - c1.execute('create database if not exists db;'); -} -catch(err) { - conn.close(); - throw err; -} - -// Now we will use database db -try { - c1.execute('use db;'); -} -catch (err) { - conn.close(); - throw err; -} - -// Let's create a table called weather -// which stores some weather data like humidity, AQI (air quality index), temperature, and some notes as text -try { - c1.execute('create table if not exists weather (ts timestamp, humidity smallint, aqi int, temperature float, notes binary(30));'); -} -catch (err) { - conn.close(); - throw err; -} - -// Let's get the description of the table weather -try { - c1.execute('describe db.weather'); -} -catch (err) { - conn.close(); - throw err; -} - -// To get results, we run the function c1.fetchall() -// It only returns the query results as an array of result rows, but also stores the latest results in c1.data -try { - var tableDesc = c1.fetchall(); // The description variable here is equal to c1.data; - console.log(tableDesc); -} -catch (err) { - conn.close(); - throw err; -} - -// Let's try to insert some random generated data to test with - -let stime = new Date(); -let interval = 1000; - -// Timestamps must be in the form of "YYYY-MM-DD HH:MM:SS.MMM" if they are in milliseconds -// "YYYY-MM-DD HH:MM:SS.MMMMMM" if they are in microseconds -// Thus, we create the following function to convert a javascript Date object to the correct formatting -function convertDateToTS(date) { - let tsArr = date.toISOString().split("T") - return "\"" + tsArr[0] + " " + tsArr[1].substring(0, tsArr[1].length-1) + "\""; -} - -try { - for (let i = 0; i < 10000; i++) { - stime.setMilliseconds(stime.getMilliseconds() + interval); - let insertData = [convertDateToTS(stime), - parseInt(Math.random()*100), - parseInt(Math.random()*300), - parseFloat(Math.random()*10 + 30), - "\"random note!\""]; - c1.execute('insert into db.weather values(' + insertData.join(',') + ' );'); - } -} -catch (err) { - conn.close(); - throw err; -} - -// Now let's look at our newly inserted data -var retrievedData; -try { - c1.execute('select * from db.weather;') - retrievedData = c1.fetchall(); - - // c1.fields stores the names of each column retrieved - console.log(c1.fields); - console.log(retrievedData); - // timestamps retrieved are always JS Date Objects - // Numbers are numbers, big ints are big ints, and strings are strings -} -catch (err) { - conn.close(); - throw err; -} - -// Let's try running some basic functions -try { - c1.execute('select count(*), avg(temperature), max(temperature), min(temperature), stddev(temperature) from db.weather;') - c1.fetchall(); - console.log(c1.fields); - console.log(c1.data); -} -catch(err) { - conn.close(); - throw err; -} - -conn.close(); - -// Feel free to fork this repository or copy this code and start developing your own apps and backends with NodeJS and TDengine! diff --git a/examples/nodejs/node-example.js b/examples/nodejs/node-example.js deleted file mode 100644 index bfdd9e49a08fc1d6c20f3bdafbb84a500986b9fd..0000000000000000000000000000000000000000 --- a/examples/nodejs/node-example.js +++ /dev/null @@ -1,153 +0,0 @@ -/* This example is to show the preferred way to use the td-connector */ -/* To run, enter node path/to/node-example.js */ -// Get the td-connector package -const taos = require('td2.0-connector'); - -/* We will connect to TDengine by passing an object comprised of connection options to taos.connect and store the - * connection to the variable conn - */ -/* - * Connection Options - * host: the host to connect to - * user: the use to login as - * password: the password for the above user to login - * config: the location of the taos.cfg file, by default it is in /etc/taos - * port: the port we connect through - */ -var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0}); - -// Initialize our TDengineCursor, which we use to interact with TDengine -var c1 = conn.cursor(); - -// c1.query(query) will return a TaosQuery object, of which then we can execute. The execute function then returns a promise -// Let's create a database named db -try { - c1.execute('create database if not exists db;'); - //var query = c1.query('create database if not exists db;'); - //query.execute(); -} -catch(err) { - conn.close(); - throw err; -} - -// Now we will use database db. As this query won't return any results, -// we can simplify the code and directly use the c1.execute() function. No need for a TaosQuery object to wrap around the query -try { - c1.execute('use db;'); -} -catch (err) { - conn.close(); - throw err; -} - -// Let's create a table called weather -// which stores some weather data like humidity, AQI (air quality index), temperature, and some notes as text -// We can also immedietely execute a TaosQuery object by passing true as the secodn argument -// This will then return a promise that we can then attach a callback function to -try { - var promise = c1.query('create table if not exists weather (ts timestamp, humidity smallint, aqi int, temperature float, notes binary(30));', true); - promise.then(function(){ - console.log("Table created!"); - }).catch(function() { - console.log("Table couldn't be created.") - }); -} -catch (err) { - conn.close(); - throw err; -} - -// Let's get the description of the table weather -// When using a TaosQuery object and then executing it, upon success it returns a TaosResult object, which is a wrapper around the -// retrieved data and allows us to easily access data and manipulate or display it. -try { - c1.query('describe db.weather;').execute().then(function(result){ - // Result is an instance of TaosResult and has the function pretty() which instantly logs a prettified version to the console - result.pretty(); - }); -} -catch (err) { - conn.close(); - throw err; -} - - -Date.prototype.Format = function(fmt){ - var o = { - 'M+': this.getMonth() + 1, - 'd+': this.getDate(), - 'H+': this.getHours(), - 'm+': this.getMinutes(), - 's+': this.getSeconds(), - 'S+': this.getMilliseconds() - }; - if (/(y+)/.test(fmt)) { - fmt = fmt.replace(RegExp.$1, (this.getFullYear() + '').substr(4 - RegExp.$1.length)); - } - for (var k in o) { - if (new RegExp('(' + k + ')').test(fmt)) { - fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (('00' + o[k]).substr(String(o[k]).length))); - } - } - return fmt; -} - - -// Let's try to insert some random generated data to test with -// We will use the bind function of the TaosQuery object to easily bind values to question marks in the query -// For Timestamps, a normal Datetime object or TaosTimestamp or milliseconds can be passed in through the bind function -let stime = new Date(); -let interval = 1000; -try { - for (let i = 0; i < 1000; i++) { - stime.setMilliseconds(stime.getMilliseconds() + interval); - - //console.log(stime.Format('yyyy-MM-dd HH:mm:ss.SSS')); - - let insertData = [stime, - parseInt(Math.random()*100), - parseInt(Math.random()*300), - parseFloat(Math.random()*10 + 30), - "Note"]; - //c1.execute('insert into db.weather values(' + insertData.join(',') + ' );'); - - //var query = c1.query('insert into db.weather values(?, ?, ?, ?, ?);').bind(insertData); - //query.execute(); - c1.execute('insert into db.weather values(\"'+stime.Format('yyyy-MM-dd HH:mm:ss.SSS')+'\",'+parseInt(Math.random() * 100)+','+parseInt(Math.random() * 300)+','+parseFloat(Math.random()*10 + 30)+',"Note");'); - } -}catch (err) { - conn.close(); - throw err; -} - -// Now let's look at our newly inserted data -var retrievedData; -try { - c1.query('select * from db.weather limit 5 offset 100;', true).then(function(result){ - //result.pretty(); - console.log('=========>'+JSON.stringify(result)); - // Neat! - }); - -} -catch (err) { - conn.close(); - throw err; -} - -// Let's try running some basic functions -try { - c1.query('select count(*), avg(temperature), max(temperature), min(temperature), stddev(temperature) from db.weather;', true) - .then(function(result) { - result.pretty(); - }) -} -catch(err) { - conn.close(); - throw err; -} - -conn.close(); - -// Feel free to fork this repository or copy this code and start developing your own apps and backends with NodeJS and TDengine! diff --git a/examples/nodejs/nodejsChecker.js b/examples/nodejs/nodejsChecker.js index e634a54ea1c56310bae7d38188590a0d7862f953..58b27dad67ba578b3b53194d3de73373991c2240 100644 --- a/examples/nodejs/nodejsChecker.js +++ b/examples/nodejs/nodejsChecker.js @@ -1,29 +1,27 @@ -const taos = require('td2.0-connector'); -//const taos = require('../../../src/connector/nodejs/'); - +const taos = require("@tdengine/client"); var host = null; var port = 6030; for(var i = 2; i < global.process.argv.length; i++){ - var key = global.process.argv[i].split("=")[0]; - var value = global.process.argv[i].split("=")[1]; - - if("host" == key){ - host = value; - } - if("port" == key){ - port = value; - } + var key = global.process.argv[i].split("=")[0]; + var value = global.process.argv[i].split("=")[1]; + + if("host" == key){ + host = value; + } + if("port" == key){ + port = value; + } } if(host == null){ - console.log("Usage: node nodejsChecker.js host= port="); - process.exit(0); + console.log("Usage: node nodejsChecker.js host= port="); + process.exit(0); } // establish connection var conn = taos.connect({host:host, user:"root", password:"taosdata",port:port}); -var cursor = conn.cursor(); +var cursor = conn.cursor(); // create database executeSql("create database if not exists test", 0); // use db @@ -40,22 +38,22 @@ executeQuery("select * from test.weather"); conn.close(); function executeQuery(sql){ - var start = new Date().getTime(); - var promise = cursor.query(sql, true); - var end = new Date().getTime(); - promise.then(function(result){ - printSql(sql, result != null,(end - start)); - result.pretty(); - }); + var start = new Date().getTime(); + var promise = cursor.query(sql, true); + var end = new Date().getTime(); + promise.then(function(result){ + printSql(sql, result != null,(end - start)); + result.pretty(); + }); } function executeSql(sql, affectRows){ - var start = new Date().getTime(); - var promise = cursor.execute(sql); - var end = new Date().getTime(); - printSql(sql, promise == affectRows, (end - start)); + var start = new Date().getTime(); + var promise = cursor.execute(sql); + var end = new Date().getTime(); + printSql(sql, promise == affectRows, (end - start)); } function printSql(sql, succeed, cost){ - console.log("[ "+(succeed ? "OK" : "ERROR!")+" ] time cost: " + cost + " ms, execute statement ====> " + sql); -} + console.log("[ "+(succeed ? "OK" : "ERROR!")+" ] time cost: " + cost + " ms, execute statement ====> " + sql); +} \ No newline at end of file diff --git a/examples/nodejs/test1970.js b/examples/nodejs/test1970.js deleted file mode 100644 index 5177a7371e9a07fa7b548936ff038c1f2a29bc1f..0000000000000000000000000000000000000000 --- a/examples/nodejs/test1970.js +++ /dev/null @@ -1,125 +0,0 @@ -const taos = require('td2.0-connector'); -var conn = taos.connect({host:"127.0.0.1", user:"root", password:"taosdata", config:"/etc/taos",port:0}) -var c1 = conn.cursor(); // Initializing a new cursor - -let stime = new Date(); -let interval = 1000; - -function convertDateToTS(date) { - let tsArr = date.toISOString().split("T") - return "\"" + tsArr[0] + " " + tsArr[1].substring(0, tsArr[1].length - 1) + "\""; -} - -function R(l, r) { - return Math.random() * (r - l) - r; -} - -function randomBool() { - if (Math.random() < 0.5) { - return true; - } - return false; -} - -// Initialize -const dbname = "nodejs_1970_db"; -const tbname = "t1"; - -let dropDB = "drop database if exists " + dbname -console.log(dropDB);//asdasdasd -c1.execute(dropDB);///asdasd - -let createDB = "create database " + dbname + " keep 36500" -console.log(createDB); -c1.execute(createDB); - -let useTbl = "use " + dbname -console.log(useTbl) -c1.execute(useTbl); - -let createTbl = "create table if not exists " + tbname + "(ts timestamp,id int)" -console.log(createTbl); -c1.execute(createTbl); - -//1969-12-31 23:59:59.999 -//1970-01-01 00:00:00.000 -//1970-01-01 07:59:59.999 -//1970-01-01 08:00:00.000a -//1628928479484 2021-08-14 08:07:59.484 -let sql1 = "insert into " + dbname + "." + tbname + " values('1969-12-31 23:59:59.999',1)" -console.log(sql1); -c1.execute(sql1); - -let sql2 = "insert into " + dbname + "." + tbname + " values('1970-01-01 00:00:00.000',2)" -console.log(sql2); -c1.execute(sql2); - -let sql3 = "insert into " + dbname + "." + tbname + " values('1970-01-01 07:59:59.999',3)" -console.log(sql3); -c1.execute(sql3); - -let sql4 = "insert into " + dbname + "." + tbname + " values('1970-01-01 08:00:00.000',4)" -console.log(sql4); -c1.execute(sql4); - -let sql5 = "insert into " + dbname + "." + tbname + " values('2021-08-14 08:07:59.484',5)" -console.log(sql5); -c1.execute(sql5); - -// Select -let query1 = "select * from " + dbname + "." + tbname -console.log(query1); -c1.execute(query1); - -var d = c1.fetchall(); -console.log(c1.fields); -for (let i = 0; i < d.length; i++) - console.log(d[i][0].valueOf()); - -//initialize -let initSql1 = "drop table if exists " + tbname -console.log(initSql1); -c1.execute(initSql1); - -console.log(createTbl); -c1.execute(createTbl); -c1.execute(useTbl) - -//-28800001 1969-12-31 23:59:59.999 -//-28800000 1970-01-01 00:00:00.000 -//-1 1970-01-01 07:59:59.999 -//0 1970-01-01 08:00:00.00 -//1628928479484 2021-08-14 08:07:59.484 -let sql11 = "insert into " + dbname + "." + tbname + " values(-28800001,11)"; -console.log(sql11); -c1.execute(sql11); - -let sql12 = "insert into " + dbname + "." + tbname + " values(-28800000,12)" -console.log(sql12); -c1.execute(sql12); - -let sql13 = "insert into " + dbname + "." + tbname + " values(-1,13)" -console.log(sql13); -c1.execute(sql13); - -let sql14 = "insert into " + dbname + "." + tbname + " values(0,14)" -console.log(sql14); -c1.execute(sql14); - -let sql15 = "insert into " + dbname + "." + tbname + " values(1628928479484,15)" -console.log(sql15); -c1.execute(sql15); - -// Select -console.log(query1); -c1.execute(query1); - -var d = c1.fetchall(); -console.log(c1.fields); -for (let i = 0; i < d.length; i++) - console.log(d[i][0].valueOf()); - -setTimeout(function () { - conn.close(); -}, 2000); - diff --git a/include/client/taos.h b/include/client/taos.h index 838d0e826662abe5d2fbd6253601a12f06978c75..cf410a42daf1e9c401af767497a603aa12c7a536 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -208,6 +208,7 @@ DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); +DLL_EXPORT int taos_get_current_db(TAOS *taos, char *database, int len, int *required); DLL_EXPORT const char *taos_errstr(TAOS_RES *res); DLL_EXPORT int taos_errno(TAOS_RES *res); diff --git a/include/common/systable.h b/include/common/systable.h index cfc0af0172a612c16a52d68fee8705e615d32602..558a1ca297bfd42269fd376c5e6500e1e4bdfbcc 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -36,6 +36,7 @@ extern "C" { #define TSDB_INS_TABLE_STABLES "ins_stables" #define TSDB_INS_TABLE_TABLES "ins_tables" #define TSDB_INS_TABLE_TAGS "ins_tags" +#define TSDB_INS_TABLE_COLS "ins_columns" #define TSDB_INS_TABLE_TABLE_DISTRIBUTED "ins_table_distributed" #define TSDB_INS_TABLE_USERS "ins_users" #define TSDB_INS_TABLE_LICENCES "ins_grants" diff --git a/include/common/taosdef.h b/include/common/taosdef.h index bf4de9d4ded1d0955bef05b1e3000be0bf34d8aa..d1ca446904594fa57d54031d5b39e529795d70da 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -30,6 +30,11 @@ typedef int64_t tb_uid_t; #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) #define TSWINDOW_IS_EQUAL(t1, t2) (((t1).skey == (t2).skey) && ((t1).ekey == (t2).ekey)) +//define show cluster alive and show db.alive +#define SHOW_STATUS_NOT_AVAILABLE 0 +#define SHOW_STATUS_AVAILABLE 1 +#define SHOW_STATUS_HALF_AVAILABLE 2 + typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 72d969a0d54be5c547d984e5b80750093180fcd7..2a40976a8b9a21e47ca49b0d62193f2c8fcff306 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -162,6 +162,7 @@ typedef enum EStreamType { STREAM_PULL_DATA, STREAM_PULL_OVER, STREAM_FILL_OVER, + STREAM_CREATE_CHILD_TABLE, } EStreamType; #pragma pack(push, 1) @@ -204,8 +205,7 @@ typedef struct SDataBlockInfo { STimeWindow calWin; // used for stream, do not serialize TSKEY watermark; // used for stream - char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition - STag* pTag; // used for stream partition + char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition } SDataBlockInfo; typedef struct SSDataBlock { @@ -341,7 +341,7 @@ typedef struct SExprInfo { typedef struct { const char* key; - int32_t keyLen; + size_t keyLen; uint8_t type; union { const char* value; @@ -350,7 +350,7 @@ typedef struct { double d; float f; }; - int32_t length; + size_t length; } SSmlKv; #define QUERY_ASC_FORWARD_STEP 1 @@ -384,6 +384,11 @@ typedef struct STUidTagInfo { #define CALCULATE_END_TS_COLUMN_INDEX 5 #define TABLE_NAME_COLUMN_INDEX 6 +// stream create table block column +#define UD_TABLE_NAME_COLUMN_INDEX 0 +#define UD_GROUPID_COLUMN_INDEX 1 +#define UD_TAG_COLUMN_INDEX 2 + #ifdef __cplusplus } #endif diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 75ee878ea2d47f90bc1848606a9eab3fef947f9d..84d696e518c45c65d7fbb2f6d87689a656121dab 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -243,7 +243,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid); char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6855287fb2d354f52998342d368c46b3a852bf89..8be5cb4d413b9392702c628d72f27cb0662687a3 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -29,6 +29,7 @@ extern "C" { typedef struct SBuffer SBuffer; typedef struct SSchema SSchema; +typedef struct SSchema2 SSchema2; typedef struct STColumn STColumn; typedef struct STSchema STSchema; typedef struct SValue SValue; @@ -44,18 +45,38 @@ typedef struct SColData SColData; #define HAS_VALUE ((uint8_t)0x4) // bitmap ================================ -const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, - {0b00000000, 0b00000100, 0b00001000, 2}, - {0b00000000, 0b00010000, 0b00100000, 4}, - {0b00000000, 0b01000000, 0b10000000, 6}}; - -#define N1(n) ((((uint8_t)1) << (n)) - 1) -#define BIT1_SIZE(n) ((((n)-1) >> 3) + 1) -#define BIT2_SIZE(n) ((((n)-1) >> 2) + 1) -#define SET_BIT1(p, i, v) ((p)[(i) >> 3] = (p)[(i) >> 3] & N1((i)&7) | (((uint8_t)(v)) << ((i)&7))) -#define GET_BIT1(p, i) (((p)[(i) >> 3] >> ((i)&7)) & ((uint8_t)1)) -#define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) -#define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) +const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, + 0b11101111, 0b11011111, 0b10111111, 0b01111111}; + +const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; + +#define ONE ((uint8_t)1) +#define THREE ((uint8_t)3) +#define DIV_8(i) ((i) >> 3) +#define MOD_8(i) ((i)&7) +#define DIV_4(i) ((i) >> 2) +#define MOD_4(i) ((i)&3) +#define MOD_4_TIME_2(i) (MOD_4(i) << 1) +#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) +#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) +#define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) +#define SET_BIT1_EX(p, i, v) \ + do { \ + if (MOD_8(i) == 0) { \ + (p)[DIV_8(i)] = 0; \ + } \ + SET_BIT1(p, i, v); \ + } while (0) +#define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) +#define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) +#define SET_BIT2_EX(p, i, v) \ + do { \ + if (MOD_4(i) == 0) { \ + (p)[DIV_4(i)] = 0; \ + } \ + SET_BIT2(p, i, v); \ + } while (0) +#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) // SBuffer ================================ struct SBuffer { @@ -70,9 +91,6 @@ int32_t tBufferInit(SBuffer *pBuffer, int64_t size); int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); -// STSchema ================================ -void tDestroyTSchema(STSchema *pTSchema); - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) @@ -87,8 +105,12 @@ void tDestroyTSchema(STSchema *pTSchema); #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) // SRow ================================ -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer); -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow); +int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); +void tRowDestroy(SRow *pRow); +void tRowSort(SArray *aRowP); +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); +int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); @@ -110,15 +132,29 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remov int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); // SColData ================================ +typedef void *(*xMallocFn)(void *, int32_t); void tColDataDestroy(void *ph); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataClear(SColData *pColData); +void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); +int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward); void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); extern void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull); +// for stmt bind +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind); +void tColDataSortMerge(SArray *colDataArr); + +// for raw block +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, + char *data); +// for encode/decode +int32_t tPutColData(uint8_t *pBuf, SColData *pColData); +int32_t tGetColData(uint8_t *pBuf, SColData *pColData); + // STRUCT ================================ struct STColumn { col_id_t colId; @@ -166,8 +202,11 @@ struct SColData { int16_t cid; int8_t type; int8_t smaOn; + int32_t numOfNone; // # of none + int32_t numOfNull; // # of null + int32_t numOfValue; // # of vale int32_t nVal; - uint8_t flag; + int8_t flag; uint8_t *pBitMap; int32_t *aOffset; int32_t nData; @@ -225,23 +264,15 @@ struct STag { memcpy(varDataVal(x), (str), (_size)); \ } while (0); -// ----------------- SCHEMA BUILDER DEFINITION -typedef struct { - int32_t tCols; - int32_t nCols; - schema_ver_t version; - uint16_t flen; - int32_t tlen; - STColumn *columns; -} STSchemaBuilder; - -int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes); -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); - +// STSchema ================================ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); +#define tDestroyTSchema(pTSchema) \ + do { \ + if (pTSchema) { \ + taosMemoryFree(pTSchema); \ + pTSchema = NULL; \ + } \ + } while (0) #endif diff --git a/include/common/tglobal.h b/include/common/tglobal.h index e92afc22221a4f647bf6ee5a2a4e5c089e856bfd..b5fd6c0270a0cb84d15d298379afb81968b4cf56 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -142,8 +142,8 @@ extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; extern char tsSmlTagName[]; -extern bool tsSmlDataFormat; -extern int32_t tsSmlBatchSize; +//extern bool tsSmlDataFormat; +//extern int32_t tsSmlBatchSize; // wal extern int64_t tsWalFsyncDataSizeLimit; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 335c57afd36af2ce7a46a89e3a4cfdc479c9af76..cd63f7d27800a89e6903d37740d54a44806e7c19 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -115,6 +115,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STREAMS, TSDB_MGMT_TABLE_TABLE, TSDB_MGMT_TABLE_TAG, + TSDB_MGMT_TABLE_COL, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VGROUP, @@ -143,6 +144,8 @@ typedef enum _mgmt_table { #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 +#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11 +#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -293,6 +296,15 @@ struct SSchema { char name[TSDB_COL_NAME_LEN]; }; +struct SSchema2 { + int8_t type; + int8_t flags; + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; + char alias[TSDB_COL_NAME_LEN]; +}; + typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; @@ -345,7 +357,19 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) -#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_IDX_ON(s) (((s)->flags & 0x02) == COL_IDX_ON) +#define IS_SET_NULL(s) (((s)->flags & COL_SET_NULL) == COL_SET_NULL) + +#define SSCHMEA_SET_IDX_ON(s) \ + do { \ + (s)->flags |= COL_IDX_ON; \ + } while (0) + +#define SSCHMEA_SET_IDX_OFF(s) \ + do { \ + (s)->flags &= (~COL_IDX_ON); \ + } while (0) #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) @@ -383,6 +407,13 @@ static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { } } +static FORCE_INLINE void tDeleteSSchemaWrapperForHash(void* pSchemaWrapper) { + if (pSchemaWrapper != NULL && *(SSchemaWrapper**)pSchemaWrapper != NULL) { + taosMemoryFree((*(SSchemaWrapper**)pSchemaWrapper)->pSchema); + taosMemoryFree(*(SSchemaWrapper**)pSchemaWrapper); + } +} + static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); @@ -484,8 +515,6 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW return 0; } -STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver); - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; @@ -1234,9 +1263,20 @@ int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); typedef struct { + char colName[TSDB_COL_NAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + int64_t stbUid; int64_t dbUid; - char db[TSDB_DB_FNAME_LEN]; int64_t reserved[8]; +} SDropIndexReq; + +int32_t tSerializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq); +int32_t tDeserializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq); + +typedef struct { + int64_t dbUid; + char db[TSDB_DB_FNAME_LEN]; + int64_t compactStartTime; } SCompactVnodeReq; int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); @@ -1274,6 +1314,25 @@ typedef struct { int32_t tSerializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); int32_t tDeserializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); +typedef struct { + int32_t vgId; + int8_t disable; +} SDisableVnodeWriteReq; + +int32_t tSerializeSDisableVnodeWriteReq(void* buf, int32_t bufLen, SDisableVnodeWriteReq* pReq); +int32_t tDeserializeSDisableVnodeWriteReq(void* buf, int32_t bufLen, SDisableVnodeWriteReq* pReq); + +typedef struct { + int32_t srcVgId; + int32_t dstVgId; + uint32_t hashBegin; + uint32_t hashEnd; + int64_t reserved; +} SAlterVnodeHashRangeReq; + +int32_t tSerializeSAlterVnodeHashRangeReq(void* buf, int32_t bufLen, SAlterVnodeHashRangeReq* pReq); +int32_t tDeserializeSAlterVnodeHashRangeReq(void* buf, int32_t bufLen, SAlterVnodeHashRangeReq* pReq); + typedef struct { SMsgHead header; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1397,6 +1456,7 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; char user[TSDB_USER_LEN]; + char filterTb[TSDB_TABLE_NAME_LEN]; int64_t showId; } SRetrieveTableReq; @@ -1737,6 +1797,8 @@ typedef struct { int32_t execId; } STaskDropReq; +int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); +int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tSerializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); int32_t tDeserializeSTaskDropReq(void* buf, int32_t bufLen, STaskDropReq* pReq); @@ -1755,6 +1817,14 @@ typedef struct { #define STREAM_FILL_HISTORY_OFF 0 #define STREAM_DEFAULT_FILL_HISTORY STREAM_FILL_HISTORY_OFF #define STREAM_DEFAULT_IGNORE_UPDATE 0 +#define STREAM_CREATE_STABLE_TRUE 1 +#define STREAM_CREATE_STABLE_FALSE 0 + +typedef struct SColLocation { + int16_t slotId; + col_id_t colId; + int8_t type; +} SColLocation; typedef struct { char name[TSDB_STREAM_FNAME_LEN]; @@ -1772,8 +1842,12 @@ typedef struct { SArray* pTags; // array of SField // 3.0.20 int64_t checkpointFreq; // ms - int64_t deleteMark; - int8_t igUpdate; + // 3.0.2.3 + int8_t createStb; + uint64_t targetStbUid; + SArray* fillNullCols; // array of SColLocation + int64_t deleteMark; + int8_t igUpdate; } SCMCreateStreamReq; typedef struct { @@ -2087,10 +2161,15 @@ typedef struct SVCreateTbReq { }; } SVCreateTbReq; -int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); -int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); +int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +void tDestroySVCreateTbReq(SVCreateTbReq* pReq, int32_t flags); static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { + if (NULL == req) { + return; + } + taosMemoryFreeClear(req->name); taosMemoryFreeClear(req->comment); if (req->type == TSDB_CHILD_TABLE) { @@ -2767,6 +2846,22 @@ typedef struct { int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +typedef struct { + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + char idxName[TSDB_INDEX_FNAME_LEN]; + int8_t idxType; +} SCreateTagIndexReq; + +int32_t tSerializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexReq* pReq); +int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexReq* pReq); + +typedef SMDropSmaReq SDropTagIndexReq; + +int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX @@ -3238,6 +3333,64 @@ int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tSerializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tDeserializeSMqAskEpReq(void* buf, int32_t bufLen, SMqAskEpReq* pReq); +int32_t tSerializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); +int32_t tDeserializeSMqHbReq(void* buf, int32_t bufLen, SMqHbReq* pReq); + +#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 +#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 + +typedef struct { + int32_t flags; + SVCreateTbReq* pCreateTbReq; + int64_t suid; + int64_t uid; + int32_t sver; + union { + SArray* aRowP; + SArray* aCol; + }; +} SSubmitTbData; + +typedef struct { + SArray* aSubmitTbData; // SArray +} SSubmitReq2; + +typedef struct { + SMsgHead header; + int64_t version; + char data[]; // SSubmitReq2 +} SSubmitReq2Msg; + +int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); +int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); +void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); + +typedef struct { + int32_t affectedRows; + SArray* aCreateTbRsp; // SArray +} SSubmitRsp2; + +int32_t tEncodeSSubmitRsp2(SEncoder* pCoder, const SSubmitRsp2* pRsp); +int32_t tDecodeSSubmitRsp2(SDecoder* pCoder, SSubmitRsp2* pRsp); +void tDestroySSubmitRsp2(SSubmitRsp2* pRsp, int32_t flag); + +#define TSDB_MSG_FLG_ENCODE 0x1 +#define TSDB_MSG_FLG_DECODE 0x2 +#define TSDB_MSG_FLG_CMPT 0x3 + +typedef struct { + union { + struct { + void* msgStr; + int32_t msgLen; + int64_t ver; + }; + void* pDataBlock; + }; +} SPackedData; #pragma pack(pop) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index eaac319141521a7e80f2f64a02e5580f7375bd74..eca8740d2864612ff604601d089b206690854290 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -39,7 +39,7 @@ typedef enum { QUEUE_MAX, } EQueueType; -typedef void (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); +typedef bool (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg); typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype); typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg); @@ -70,7 +70,7 @@ void tmsgSendRsp(SRpcMsg* pMsg); void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg); void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type); void tmsgReportStartup(const char* name, const char* desc); -void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); +bool tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port); void tmsgUpdateDnodeEpSet(SEpSet* epset); #ifdef __cplusplus diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 7833bdf1393c72b988b1553d691e7d8936b14603..46ca814e50ac6d33f24456e6ed9ecb7e8769dd41 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -172,8 +172,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SERVER_VERSION, "server-version", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) + // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) + // TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) @@ -220,6 +220,10 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) + + TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_NEW_MSG_SEG(TDMT_SCH_MSG) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 1d69e129f4110d4f461ed1b0d65cce578551b2e4..ada3fd20e8ce693a7fcd9a331d24b968fe1852c4 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -77,268 +77,272 @@ #define TK_USE 59 #define TK_FLUSH 60 #define TK_TRIM 61 -#define TK_IF 62 -#define TK_NOT 63 -#define TK_EXISTS 64 -#define TK_BUFFER 65 -#define TK_CACHEMODEL 66 -#define TK_CACHESIZE 67 -#define TK_COMP 68 -#define TK_DURATION 69 -#define TK_NK_VARIABLE 70 -#define TK_MAXROWS 71 -#define TK_MINROWS 72 -#define TK_KEEP 73 -#define TK_PAGES 74 -#define TK_PAGESIZE 75 -#define TK_TSDB_PAGESIZE 76 -#define TK_PRECISION 77 -#define TK_REPLICA 78 -#define TK_VGROUPS 79 -#define TK_SINGLE_STABLE 80 -#define TK_RETENTIONS 81 -#define TK_SCHEMALESS 82 -#define TK_WAL_LEVEL 83 -#define TK_WAL_FSYNC_PERIOD 84 -#define TK_WAL_RETENTION_PERIOD 85 -#define TK_WAL_RETENTION_SIZE 86 -#define TK_WAL_ROLL_PERIOD 87 -#define TK_WAL_SEGMENT_SIZE 88 -#define TK_STT_TRIGGER 89 -#define TK_TABLE_PREFIX 90 -#define TK_TABLE_SUFFIX 91 -#define TK_NK_COLON 92 -#define TK_MAX_SPEED 93 -#define TK_TABLE 94 -#define TK_NK_LP 95 -#define TK_NK_RP 96 -#define TK_STABLE 97 -#define TK_ADD 98 -#define TK_COLUMN 99 -#define TK_MODIFY 100 -#define TK_RENAME 101 -#define TK_TAG 102 -#define TK_SET 103 -#define TK_NK_EQ 104 -#define TK_USING 105 -#define TK_TAGS 106 -#define TK_COMMENT 107 -#define TK_BOOL 108 -#define TK_TINYINT 109 -#define TK_SMALLINT 110 -#define TK_INT 111 -#define TK_INTEGER 112 -#define TK_BIGINT 113 -#define TK_FLOAT 114 -#define TK_DOUBLE 115 -#define TK_BINARY 116 -#define TK_TIMESTAMP 117 -#define TK_NCHAR 118 -#define TK_UNSIGNED 119 -#define TK_JSON 120 -#define TK_VARCHAR 121 -#define TK_MEDIUMBLOB 122 -#define TK_BLOB 123 -#define TK_VARBINARY 124 -#define TK_DECIMAL 125 -#define TK_MAX_DELAY 126 -#define TK_WATERMARK 127 -#define TK_ROLLUP 128 -#define TK_TTL 129 -#define TK_SMA 130 -#define TK_DELETE_MARK 131 -#define TK_FIRST 132 -#define TK_LAST 133 -#define TK_SHOW 134 -#define TK_PRIVILEGES 135 -#define TK_DATABASES 136 -#define TK_TABLES 137 -#define TK_STABLES 138 -#define TK_MNODES 139 -#define TK_QNODES 140 -#define TK_FUNCTIONS 141 -#define TK_INDEXES 142 -#define TK_ACCOUNTS 143 -#define TK_APPS 144 -#define TK_CONNECTIONS 145 -#define TK_LICENCES 146 -#define TK_GRANTS 147 -#define TK_QUERIES 148 -#define TK_SCORES 149 -#define TK_TOPICS 150 -#define TK_VARIABLES 151 -#define TK_CLUSTER 152 -#define TK_BNODES 153 -#define TK_SNODES 154 -#define TK_TRANSACTIONS 155 -#define TK_DISTRIBUTED 156 -#define TK_CONSUMERS 157 -#define TK_SUBSCRIPTIONS 158 -#define TK_VNODES 159 -#define TK_LIKE 160 -#define TK_TBNAME 161 -#define TK_QTAGS 162 -#define TK_AS 163 -#define TK_INDEX 164 -#define TK_FUNCTION 165 -#define TK_INTERVAL 166 -#define TK_TOPIC 167 -#define TK_WITH 168 -#define TK_META 169 -#define TK_CONSUMER 170 -#define TK_GROUP 171 -#define TK_DESC 172 -#define TK_DESCRIBE 173 -#define TK_RESET 174 -#define TK_QUERY 175 -#define TK_CACHE 176 -#define TK_EXPLAIN 177 -#define TK_ANALYZE 178 -#define TK_VERBOSE 179 -#define TK_NK_BOOL 180 -#define TK_RATIO 181 -#define TK_NK_FLOAT 182 -#define TK_OUTPUTTYPE 183 -#define TK_AGGREGATE 184 -#define TK_BUFSIZE 185 -#define TK_STREAM 186 -#define TK_INTO 187 -#define TK_TRIGGER 188 -#define TK_AT_ONCE 189 -#define TK_WINDOW_CLOSE 190 -#define TK_IGNORE 191 -#define TK_EXPIRED 192 -#define TK_FILL_HISTORY 193 -#define TK_UPDATE 194 -#define TK_SUBTABLE 195 -#define TK_KILL 196 -#define TK_CONNECTION 197 -#define TK_TRANSACTION 198 -#define TK_BALANCE 199 -#define TK_VGROUP 200 -#define TK_MERGE 201 -#define TK_REDISTRIBUTE 202 -#define TK_SPLIT 203 -#define TK_DELETE 204 -#define TK_INSERT 205 -#define TK_NULL 206 -#define TK_NK_QUESTION 207 -#define TK_NK_ARROW 208 -#define TK_ROWTS 209 -#define TK_QSTART 210 -#define TK_QEND 211 -#define TK_QDURATION 212 -#define TK_WSTART 213 -#define TK_WEND 214 -#define TK_WDURATION 215 -#define TK_IROWTS 216 -#define TK_CAST 217 -#define TK_NOW 218 -#define TK_TODAY 219 -#define TK_TIMEZONE 220 -#define TK_CLIENT_VERSION 221 -#define TK_SERVER_VERSION 222 -#define TK_SERVER_STATUS 223 -#define TK_CURRENT_USER 224 -#define TK_COUNT 225 -#define TK_LAST_ROW 226 -#define TK_CASE 227 -#define TK_END 228 -#define TK_WHEN 229 -#define TK_THEN 230 -#define TK_ELSE 231 -#define TK_BETWEEN 232 -#define TK_IS 233 -#define TK_NK_LT 234 -#define TK_NK_GT 235 -#define TK_NK_LE 236 -#define TK_NK_GE 237 -#define TK_NK_NE 238 -#define TK_MATCH 239 -#define TK_NMATCH 240 -#define TK_CONTAINS 241 -#define TK_IN 242 -#define TK_JOIN 243 -#define TK_INNER 244 -#define TK_SELECT 245 -#define TK_DISTINCT 246 -#define TK_WHERE 247 -#define TK_PARTITION 248 -#define TK_BY 249 -#define TK_SESSION 250 -#define TK_STATE_WINDOW 251 -#define TK_SLIDING 252 -#define TK_FILL 253 -#define TK_VALUE 254 -#define TK_VALUE_F 255 -#define TK_NONE 256 -#define TK_PREV 257 -#define TK_NULL_F 258 -#define TK_LINEAR 259 -#define TK_NEXT 260 -#define TK_HAVING 261 -#define TK_RANGE 262 -#define TK_EVERY 263 -#define TK_ORDER 264 -#define TK_SLIMIT 265 -#define TK_SOFFSET 266 -#define TK_LIMIT 267 -#define TK_OFFSET 268 -#define TK_ASC 269 -#define TK_NULLS 270 -#define TK_ABORT 271 -#define TK_AFTER 272 -#define TK_ATTACH 273 -#define TK_BEFORE 274 -#define TK_BEGIN 275 -#define TK_BITAND 276 -#define TK_BITNOT 277 -#define TK_BITOR 278 -#define TK_BLOCKS 279 -#define TK_CHANGE 280 -#define TK_COMMA 281 -#define TK_COMPACT 282 -#define TK_CONCAT 283 -#define TK_CONFLICT 284 -#define TK_COPY 285 -#define TK_DEFERRED 286 -#define TK_DELIMITERS 287 -#define TK_DETACH 288 -#define TK_DIVIDE 289 -#define TK_DOT 290 -#define TK_EACH 291 -#define TK_FAIL 292 -#define TK_FILE 293 -#define TK_FOR 294 -#define TK_GLOB 295 -#define TK_ID 296 -#define TK_IMMEDIATE 297 -#define TK_IMPORT 298 -#define TK_INITIALLY 299 -#define TK_INSTEAD 300 -#define TK_ISNULL 301 -#define TK_KEY 302 -#define TK_MODULES 303 -#define TK_NK_BITNOT 304 -#define TK_NK_SEMI 305 -#define TK_NOTNULL 306 -#define TK_OF 307 -#define TK_PLUS 308 -#define TK_PRIVILEGE 309 -#define TK_RAISE 310 -#define TK_REPLACE 311 -#define TK_RESTRICT 312 -#define TK_ROW 313 -#define TK_SEMI 314 -#define TK_STAR 315 -#define TK_STATEMENT 316 -#define TK_STRICT 317 -#define TK_STRING 318 -#define TK_TIMES 319 -#define TK_VALUES 320 -#define TK_VARIABLE 321 -#define TK_VIEW 322 -#define TK_WAL 323 +#define TK_COMPACT 62 +#define TK_IF 63 +#define TK_NOT 64 +#define TK_EXISTS 65 +#define TK_BUFFER 66 +#define TK_CACHEMODEL 67 +#define TK_CACHESIZE 68 +#define TK_COMP 69 +#define TK_DURATION 70 +#define TK_NK_VARIABLE 71 +#define TK_MAXROWS 72 +#define TK_MINROWS 73 +#define TK_KEEP 74 +#define TK_PAGES 75 +#define TK_PAGESIZE 76 +#define TK_TSDB_PAGESIZE 77 +#define TK_PRECISION 78 +#define TK_REPLICA 79 +#define TK_VGROUPS 80 +#define TK_SINGLE_STABLE 81 +#define TK_RETENTIONS 82 +#define TK_SCHEMALESS 83 +#define TK_WAL_LEVEL 84 +#define TK_WAL_FSYNC_PERIOD 85 +#define TK_WAL_RETENTION_PERIOD 86 +#define TK_WAL_RETENTION_SIZE 87 +#define TK_WAL_ROLL_PERIOD 88 +#define TK_WAL_SEGMENT_SIZE 89 +#define TK_STT_TRIGGER 90 +#define TK_TABLE_PREFIX 91 +#define TK_TABLE_SUFFIX 92 +#define TK_NK_COLON 93 +#define TK_MAX_SPEED 94 +#define TK_TABLE 95 +#define TK_NK_LP 96 +#define TK_NK_RP 97 +#define TK_STABLE 98 +#define TK_ADD 99 +#define TK_COLUMN 100 +#define TK_MODIFY 101 +#define TK_RENAME 102 +#define TK_TAG 103 +#define TK_SET 104 +#define TK_NK_EQ 105 +#define TK_USING 106 +#define TK_TAGS 107 +#define TK_COMMENT 108 +#define TK_BOOL 109 +#define TK_TINYINT 110 +#define TK_SMALLINT 111 +#define TK_INT 112 +#define TK_INTEGER 113 +#define TK_BIGINT 114 +#define TK_FLOAT 115 +#define TK_DOUBLE 116 +#define TK_BINARY 117 +#define TK_TIMESTAMP 118 +#define TK_NCHAR 119 +#define TK_UNSIGNED 120 +#define TK_JSON 121 +#define TK_VARCHAR 122 +#define TK_MEDIUMBLOB 123 +#define TK_BLOB 124 +#define TK_VARBINARY 125 +#define TK_DECIMAL 126 +#define TK_MAX_DELAY 127 +#define TK_WATERMARK 128 +#define TK_ROLLUP 129 +#define TK_TTL 130 +#define TK_SMA 131 +#define TK_DELETE_MARK 132 +#define TK_FIRST 133 +#define TK_LAST 134 +#define TK_SHOW 135 +#define TK_PRIVILEGES 136 +#define TK_DATABASES 137 +#define TK_TABLES 138 +#define TK_STABLES 139 +#define TK_MNODES 140 +#define TK_QNODES 141 +#define TK_FUNCTIONS 142 +#define TK_INDEXES 143 +#define TK_ACCOUNTS 144 +#define TK_APPS 145 +#define TK_CONNECTIONS 146 +#define TK_LICENCES 147 +#define TK_GRANTS 148 +#define TK_QUERIES 149 +#define TK_SCORES 150 +#define TK_TOPICS 151 +#define TK_VARIABLES 152 +#define TK_CLUSTER 153 +#define TK_BNODES 154 +#define TK_SNODES 155 +#define TK_TRANSACTIONS 156 +#define TK_DISTRIBUTED 157 +#define TK_CONSUMERS 158 +#define TK_SUBSCRIPTIONS 159 +#define TK_VNODES 160 +#define TK_ALIVE 161 +#define TK_LIKE 162 +#define TK_TBNAME 163 +#define TK_QTAGS 164 +#define TK_AS 165 +#define TK_INDEX 166 +#define TK_FUNCTION 167 +#define TK_INTERVAL 168 +#define TK_COUNT 169 +#define TK_LAST_ROW 170 +#define TK_TOPIC 171 +#define TK_WITH 172 +#define TK_META 173 +#define TK_CONSUMER 174 +#define TK_GROUP 175 +#define TK_DESC 176 +#define TK_DESCRIBE 177 +#define TK_RESET 178 +#define TK_QUERY 179 +#define TK_CACHE 180 +#define TK_EXPLAIN 181 +#define TK_ANALYZE 182 +#define TK_VERBOSE 183 +#define TK_NK_BOOL 184 +#define TK_RATIO 185 +#define TK_NK_FLOAT 186 +#define TK_OUTPUTTYPE 187 +#define TK_AGGREGATE 188 +#define TK_BUFSIZE 189 +#define TK_STREAM 190 +#define TK_INTO 191 +#define TK_TRIGGER 192 +#define TK_AT_ONCE 193 +#define TK_WINDOW_CLOSE 194 +#define TK_IGNORE 195 +#define TK_EXPIRED 196 +#define TK_FILL_HISTORY 197 +#define TK_UPDATE 198 +#define TK_SUBTABLE 199 +#define TK_KILL 200 +#define TK_CONNECTION 201 +#define TK_TRANSACTION 202 +#define TK_BALANCE 203 +#define TK_VGROUP 204 +#define TK_MERGE 205 +#define TK_REDISTRIBUTE 206 +#define TK_SPLIT 207 +#define TK_DELETE 208 +#define TK_INSERT 209 +#define TK_NULL 210 +#define TK_NK_QUESTION 211 +#define TK_NK_ARROW 212 +#define TK_ROWTS 213 +#define TK_QSTART 214 +#define TK_QEND 215 +#define TK_QDURATION 216 +#define TK_WSTART 217 +#define TK_WEND 218 +#define TK_WDURATION 219 +#define TK_IROWTS 220 +#define TK_ISFILLED 221 +#define TK_CAST 222 +#define TK_NOW 223 +#define TK_TODAY 224 +#define TK_TIMEZONE 225 +#define TK_CLIENT_VERSION 226 +#define TK_SERVER_VERSION 227 +#define TK_SERVER_STATUS 228 +#define TK_CURRENT_USER 229 +#define TK_CASE 230 +#define TK_END 231 +#define TK_WHEN 232 +#define TK_THEN 233 +#define TK_ELSE 234 +#define TK_BETWEEN 235 +#define TK_IS 236 +#define TK_NK_LT 237 +#define TK_NK_GT 238 +#define TK_NK_LE 239 +#define TK_NK_GE 240 +#define TK_NK_NE 241 +#define TK_MATCH 242 +#define TK_NMATCH 243 +#define TK_CONTAINS 244 +#define TK_IN 245 +#define TK_JOIN 246 +#define TK_INNER 247 +#define TK_SELECT 248 +#define TK_DISTINCT 249 +#define TK_WHERE 250 +#define TK_PARTITION 251 +#define TK_BY 252 +#define TK_SESSION 253 +#define TK_STATE_WINDOW 254 +#define TK_EVENT_WINDOW 255 +#define TK_START 256 +#define TK_SLIDING 257 +#define TK_FILL 258 +#define TK_VALUE 259 +#define TK_VALUE_F 260 +#define TK_NONE 261 +#define TK_PREV 262 +#define TK_NULL_F 263 +#define TK_LINEAR 264 +#define TK_NEXT 265 +#define TK_HAVING 266 +#define TK_RANGE 267 +#define TK_EVERY 268 +#define TK_ORDER 269 +#define TK_SLIMIT 270 +#define TK_SOFFSET 271 +#define TK_LIMIT 272 +#define TK_OFFSET 273 +#define TK_ASC 274 +#define TK_NULLS 275 +#define TK_ABORT 276 +#define TK_AFTER 277 +#define TK_ATTACH 278 +#define TK_BEFORE 279 +#define TK_BEGIN 280 +#define TK_BITAND 281 +#define TK_BITNOT 282 +#define TK_BITOR 283 +#define TK_BLOCKS 284 +#define TK_CHANGE 285 +#define TK_COMMA 286 +#define TK_CONCAT 287 +#define TK_CONFLICT 288 +#define TK_COPY 289 +#define TK_DEFERRED 290 +#define TK_DELIMITERS 291 +#define TK_DETACH 292 +#define TK_DIVIDE 293 +#define TK_DOT 294 +#define TK_EACH 295 +#define TK_FAIL 296 +#define TK_FILE 297 +#define TK_FOR 298 +#define TK_GLOB 299 +#define TK_ID 300 +#define TK_IMMEDIATE 301 +#define TK_IMPORT 302 +#define TK_INITIALLY 303 +#define TK_INSTEAD 304 +#define TK_ISNULL 305 +#define TK_KEY 306 +#define TK_MODULES 307 +#define TK_NK_BITNOT 308 +#define TK_NK_SEMI 309 +#define TK_NOTNULL 310 +#define TK_OF 311 +#define TK_PLUS 312 +#define TK_PRIVILEGE 313 +#define TK_RAISE 314 +#define TK_REPLACE 315 +#define TK_RESTRICT 316 +#define TK_ROW 317 +#define TK_SEMI 318 +#define TK_STAR 319 +#define TK_STATEMENT 320 +#define TK_STRICT 321 +#define TK_STRING 322 +#define TK_TIMES 323 +#define TK_VALUES 324 +#define TK_VARIABLE 325 +#define TK_VIEW 326 +#define TK_WAL 327 #define TK_NK_SPACE 600 #define TK_NK_COMMENT 601 diff --git a/include/common/ttypes.h b/include/common/ttypes.h index b7061300c1760c0b557562924caa329080779bda..97ae151b7a8ccedb84dd113fa9c9560cfe5ffe14 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -266,6 +266,7 @@ typedef struct { #define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) #define IS_INTEGER_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t))) #define IS_TIMESTAMP_TYPE(_t) ((_t) == TSDB_DATA_TYPE_TIMESTAMP) +#define IS_BOOLEAN_TYPE(_t) ((_t) == TSDB_DATA_TYPE_BOOL) #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) #define IS_MATHABLE_TYPE(_t) \ diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b3339a417ba463212c3abc163b57519194953c10..a8b1a0902acac276080d905554a3773f68e66b1c 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -22,7 +22,7 @@ typedef struct SExplainCtx SExplainCtx; -int32_t qExecCommand(bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp); int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp); int32_t qExecExplainBegin(SQueryPlan *pDag, SExplainCtx **pCtx, int64_t startTs); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 63e1c556deeae912c010c849218dec514e09be44..095d2f6d10dbe6faa0460f5532c569b543fb4ac0 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,7 +192,9 @@ int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq); +// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); +// +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); @@ -216,6 +218,7 @@ int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); int32_t qStreamRestoreParam(qTaskInfo_t tinfo); bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); +void qStreamCloseTsdbReader(void* task); #ifdef __cplusplus } diff --git a/include/libs/function/function.h b/include/libs/function/function.h index c44ad12759aa636260a46befaeaa43ce0ed13872..fb6ef26a8acf92c76cd022440a5b33ad3ca469e8 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -76,7 +76,7 @@ enum { enum { MAIN_SCAN = 0x0u, REVERSE_SCAN = 0x1u, // todo remove it - REPEAT_SCAN = 0x2u, // repeat scan belongs to the master scan + PRE_SCAN = 0x2u, // pre-scan belongs to the main scan and occurs before main scan }; typedef struct SPoint1 { diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 411c7d5e8b22640f4fd67651b3de8f7a86bb171e..7b65c06b85f990b87c0c819f1d7627d8f28e8cc0 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -120,6 +120,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_WEND, FUNCTION_TYPE_WDURATION, FUNCTION_TYPE_IROWTS, + FUNCTION_TYPE_ISFILLED, FUNCTION_TYPE_TAGS, // internal function diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index de2780691291fb2028e8f8a684f71195140a7d26..164fbf018cd39e0c7869ca95b9ba877e9488e1e4 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -40,6 +40,7 @@ extern "C" { #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) +#define SHOW_ALIVE_RESULT_COLS 1 #define PRIVILEGE_TYPE_MASK(n) (1 << n) #define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0) @@ -123,6 +124,11 @@ typedef struct STrimDatabaseStmt { int32_t maxSpeed; } STrimDatabaseStmt; +typedef struct SCompactDatabaseStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; +} SCompactDatabaseStmt; + typedef struct STableOptions { ENodeType type; bool commentNull; @@ -171,10 +177,10 @@ typedef struct SCreateSubTableClause { STableOptions* pOptions; } SCreateSubTableClause; -typedef struct SCreateMultiTableStmt { +typedef struct SCreateMultiTablesStmt { ENodeType type; SNodeList* pSubTables; -} SCreateMultiTableStmt; +} SCreateMultiTablesStmt; typedef struct SDropTableClause { ENodeType type; @@ -209,14 +215,14 @@ typedef struct SAlterTableStmt { typedef struct SCreateUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; char password[TSDB_USET_PASSWORD_LEN]; int8_t sysinfo; } SCreateUserStmt; typedef struct SAlterUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; int8_t alterType; char password[TSDB_USET_PASSWORD_LEN]; int8_t enable; @@ -225,7 +231,7 @@ typedef struct SAlterUserStmt { typedef struct SDropUserStmt { ENodeType type; - char useName[TSDB_USER_LEN]; + char userName[TSDB_USER_LEN]; } SDropUserStmt; typedef struct SCreateDnodeStmt { @@ -262,6 +268,11 @@ typedef struct SShowCreateDatabaseStmt { void* pCfg; // SDbCfgInfo } SShowCreateDatabaseStmt; +typedef struct SShowAliveStmt { + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; +} SShowAliveStmt; + typedef struct SShowCreateTableStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; @@ -295,7 +306,7 @@ typedef struct SShowTableTagsStmt { SNodeList* pTags; } SShowTableTagsStmt; -typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; +typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef struct SIndexOptions { ENodeType type; @@ -402,6 +413,7 @@ typedef struct SCreateStreamStmt { SNode* pQuery; SNodeList* pTags; SNode* pSubtable; + SNodeList* pCols; } SCreateStreamStmt; typedef struct SDropStreamStmt { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 412054b13e3bdceec6234919040c4b99a43d6102..a2e4c76adccc91d5a1f8689643f3d7a93089eba0 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -112,11 +112,12 @@ typedef enum ENodeType { QUERY_NODE_COLUMN_REF, QUERY_NODE_WHEN_THEN, QUERY_NODE_CASE_WHEN, + QUERY_NODE_EVENT_WINDOW, // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SELECT_STMT, - QUERY_NODE_VNODE_MODIF_STMT, + QUERY_NODE_VNODE_MODIFY_STMT, QUERY_NODE_CREATE_DATABASE_STMT, QUERY_NODE_DROP_DATABASE_STMT, QUERY_NODE_ALTER_DATABASE_STMT, @@ -124,7 +125,7 @@ typedef enum ENodeType { QUERY_NODE_TRIM_DATABASE_STMT, QUERY_NODE_CREATE_TABLE_STMT, QUERY_NODE_CREATE_SUBTABLE_CLAUSE, - QUERY_NODE_CREATE_MULTI_TABLE_STMT, + QUERY_NODE_CREATE_MULTI_TABLES_STMT, QUERY_NODE_DROP_TABLE_CLAUSE, QUERY_NODE_DROP_TABLE_STMT, QUERY_NODE_DROP_SUPER_TABLE_STMT, @@ -154,7 +155,7 @@ typedef enum ENodeType { QUERY_NODE_EXPLAIN_STMT, QUERY_NODE_DESCRIBE_STMT, QUERY_NODE_RESET_QUERY_CACHE_STMT, - QUERY_NODE_COMPACT_STMT, + QUERY_NODE_COMPACT_DATABASE_STMT, QUERY_NODE_CREATE_FUNCTION_STMT, QUERY_NODE_DROP_FUNCTION_STMT, QUERY_NODE_CREATE_STREAM_STMT, @@ -207,6 +208,8 @@ typedef enum ENodeType { QUERY_NODE_DELETE_STMT, QUERY_NODE_INSERT_STMT, QUERY_NODE_QUERY, + QUERY_NODE_SHOW_DB_ALIVE_STMT, + QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT, // logic plan node QUERY_NODE_LOGIC_PLAN_SCAN = 1000, @@ -265,7 +268,9 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_DELETE, QUERY_NODE_PHYSICAL_SUBPLAN, QUERY_NODE_PHYSICAL_PLAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, + QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 80fa0d3e78211464b4a327a32c8c374ba66c4953..ad4b59714cbcd83f80a4ad260004310e0740b94f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -187,7 +187,12 @@ typedef struct SMergeLogicNode { bool groupSort; } SMergeLogicNode; -typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; +typedef enum EWindowType { + WINDOW_TYPE_INTERVAL = 1, + WINDOW_TYPE_SESSION, + WINDOW_TYPE_STATE, + WINDOW_TYPE_EVENT +} EWindowType; typedef enum EWindowAlgorithm { INTERVAL_ALGO_HASH = 1, @@ -214,6 +219,8 @@ typedef struct SWindowLogicNode { SNode* pTspk; SNode* pTsEnd; SNode* pStateExpr; + SNode* pStartCond; + SNode* pEndCond; int8_t triggerType; int64_t watermark; int64_t deleteMark; @@ -503,6 +510,14 @@ typedef struct SStateWinodwPhysiNode { typedef SStateWinodwPhysiNode SStreamStateWinodwPhysiNode; +typedef struct SEventWinodwPhysiNode { + SWinodwPhysiNode window; + SNode* pStartCond; + SNode* pEndCond; +} SEventWinodwPhysiNode; + +typedef SEventWinodwPhysiNode SStreamEventWinodwPhysiNode; + typedef struct SSortPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function @@ -550,6 +565,7 @@ typedef struct SQueryInserterNode { char tableName[TSDB_TABLE_NAME_LEN]; int32_t vgId; SEpSet epSet; + bool explain; } SQueryInserterNode; typedef struct SDataDeleterNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index e291051351b433e92ee0271166a581f08fb4abfe..1a9700907effa002ce4b2acf69196eb08fd7e6d9 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -223,6 +223,13 @@ typedef struct SIntervalWindowNode { SNode* pFill; } SIntervalWindowNode; +typedef struct SEventWindowNode { + ENodeType type; // QUERY_NODE_EVENT_WINDOW + SNode* pCol; // timestamp primary key + SNode* pStartCond; + SNode* pEndCond; +} SEventWindowNode; + typedef enum EFillMode { FILL_MODE_NONE = 1, FILL_MODE_VALUE, @@ -356,34 +363,34 @@ typedef struct SVgDataBlocks { void* pData; // SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; -typedef void (*FFreeDataBlockHash)(SHashObj*); -typedef void (*FFreeDataBlockArray)(SArray*); - -typedef struct SVnodeModifOpStmt { - ENodeType nodeType; - ENodeType sqlNodeType; - SArray* pDataBlocks; // data block for each vgroup, SArray. - uint32_t insertType; // insert data from [file|sql statement| bound statement] - const char* pSql; // current sql statement position - int32_t totalRowsNum; - int32_t totalTbNum; - SName targetTableName; - SName usingTableName; - const char* pBoundCols; - struct STableMeta* pTableMeta; - SHashObj* pVgroupsHashObj; - SHashObj* pTableBlockHashObj; - SHashObj* pSubTableHashObj; - SHashObj* pTableNameHashObj; - SHashObj* pDbFNameHashObj; - SArray* pVgDataBlocks; - SVCreateTbReq createTblReq; - TdFilePtr fp; - FFreeDataBlockHash freeHashFunc; - FFreeDataBlockArray freeArrayFunc; - bool usingTableProcessing; - bool fileProcessing; -} SVnodeModifOpStmt; +typedef void (*FFreeTableBlockHash)(SHashObj*); +typedef void (*FFreeVgourpBlockArray)(SArray*); + +typedef struct SVnodeModifyOpStmt { + ENodeType nodeType; + ENodeType sqlNodeType; + SArray* pDataBlocks; // data block for each vgroup, SArray. + uint32_t insertType; // insert data from [file|sql statement| bound statement] + const char* pSql; // current sql statement position + int32_t totalRowsNum; + int32_t totalTbNum; + SName targetTableName; + SName usingTableName; + const char* pBoundCols; + struct STableMeta* pTableMeta; + SHashObj* pVgroupsHashObj; + SHashObj* pTableBlockHashObj; // SHashObj + SHashObj* pSubTableHashObj; + SHashObj* pTableNameHashObj; + SHashObj* pDbFNameHashObj; + SArray* pVgDataBlocks; // SArray + SVCreateTbReq* pCreateTblReq; + TdFilePtr fp; + FFreeTableBlockHash freeHashFunc; + FFreeVgourpBlockArray freeArrayFunc; + bool usingTableProcessing; + bool fileProcessing; +} SVnodeModifyOpStmt; typedef struct SExplainOptions { ENodeType type; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 52edd9708ca505e0bdb9be177da3f2f60b2a03a8..558203052f1e7241f1b315bb87e8710b5526db63 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -58,7 +58,6 @@ typedef struct SParseContext { bool isSuperUser; bool enableSysInfo; bool async; - int8_t schemalessType; const char* svrVer; bool nodeOffline; SArray* pTableMetaPos; // sql table pos => catalog data pos @@ -85,12 +84,13 @@ int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); void qCleanupKeywordsTable(); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); -int32_t qResetStmtDataBlock(void* block, bool keepBuf); -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc); -void qFreeStmtDataBlock(void* pDataBlock); -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId); -void qDestroyStmtDataBlock(void* pBlock); -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock); +int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb); +void qDestroyStmtDataBlock(STableDataCxt* pBlock); +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock); +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); @@ -105,12 +105,18 @@ void destroyBoundColumnInfo(void* pBoundInfo); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen); -void* smlInitHandle(SQuery* pQuery); -void smlDestroyHandle(void* pHandle); -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, - int16_t msgBufLen); -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash); +void qDestroyBoundColInfo(void* pInfo); + +SQuery* smlInitHandle(); +int32_t smlBuildRow(STableDataCxt* pTableCxt); +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* kv, int32_t index); +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta); + +int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, + STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, + char* msgBuf, int16_t msgBufLen); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); +int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD *fields, int numFields, bool needChangeLength); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f2f7ac56995ac4c516d88fddaedc55fc4aa396ca..cb547ee6b3732e391ca943bcb9b3e9783d587e36 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -164,6 +164,23 @@ typedef struct STargetInfo { int32_t vgId; } STargetInfo; +typedef struct SBoundColInfo { + int16_t* pColIndex; // bound index => schema index + int32_t numOfCols; + int32_t numOfBound; +} SBoundColInfo; + +typedef struct STableDataCxt { + STableMeta* pMeta; + STSchema* pSchema; + SBoundColInfo boundColsInfo; + SArray* pValues; + SSubmitTbData* pData; + TSKEY lastTs; + bool ordered; + bool duplicateTs; +} STableDataCxt; + typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_exec_fn_t)(void* param); @@ -239,6 +256,7 @@ int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst); void freeVgInfo(SDBVgInfo* vgInfo); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 912c09a0fb53f1c9075adaa2a57a1f5c72361965..6b09bf4899ba0f444b80449c2a0d67df160b59ef 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -35,6 +35,7 @@ typedef struct STdbState { TTB* pFillStateDb; // todo refactor TTB* pSessionStateDb; TTB* pParNameDb; + TTB* pParTagDb; TXN* txn; } STdbState; @@ -108,6 +109,9 @@ int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char* tbname); int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal); +int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen); +int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen); + #if 0 char* streamStateSessionDump(SStreamState* pState); char* streamStateIntervalDump(SStreamState* pState); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index ac09d5dfde555629ee819bba690e5ea7c25f9f9d..1d301623b1c325a7393a219140187aead37677f7 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -103,6 +103,7 @@ typedef struct { int8_t type; } SStreamQueueItem; +#if 0 typedef struct { int8_t type; int64_t ver; @@ -116,6 +117,21 @@ typedef struct { SArray* dataRefs; // SArray SArray* reqs; // SArray } SStreamMergedSubmit; +#endif + +typedef struct { + int8_t type; + int64_t ver; + int32_t* dataRef; + SPackedData submit; +} SStreamDataSubmit2; + +typedef struct { + int8_t type; + int64_t ver; + SArray* dataRefs; // SArray + SArray* submits; // SArray +} SStreamMergedSubmit2; typedef struct { int8_t type; @@ -159,20 +175,24 @@ typedef struct { void streamFreeQitem(SStreamQueueItem* data); +#if 0 bool streamQueueResEmpty(const SStreamQueueRes* pRes); int64_t streamQueueResSize(const SStreamQueueRes* pRes); SStreamQueueNode* streamQueueResFront(SStreamQueueRes* pRes); SStreamQueueNode* streamQueueResPop(SStreamQueueRes* pRes); void streamQueueResClear(SStreamQueueRes* pRes); SStreamQueueRes streamQueueBuildRes(SStreamQueueNode* pNode); +#endif typedef struct { SStreamQueueNode* pHead; } SStreamQueue1; +#if 0 bool streamQueueHasTask(const SStreamQueue1* pQueue); int32_t streamQueuePush(SStreamQueue1* pQueue, SStreamQueueItem* pItem); SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue); +#endif typedef struct { STaosQueue* queue; @@ -219,11 +239,11 @@ static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { } } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); typedef struct { char* qmsg; @@ -356,14 +376,15 @@ void tFreeSStreamTask(SStreamTask* pTask); static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { int8_t type = pItem->type; if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); + SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); if (pSubmitClone == NULL) { qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } - qDebug("task %d %p submit enqueue %p %p %p", pTask->taskId, pTask, pItem, pSubmitClone, pSubmitClone->data); + qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, + pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); // qStreamInput(pTask->exec.executor, pSubmitClone); } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || @@ -619,7 +640,7 @@ void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); -SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); +// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 3af75e0eaf443e7887dacd6a3056be4241ec5c9b..cbf1d60e35ac1c4187ddb0a5d8cc366714cbaec3 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -150,7 +150,7 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname); * @param nrname The rel name of new file. * @return int32_t 0 for success, -1 for failure. */ -int32_t tfsRename(STfs *pTfs, char *orname, char *nrname); +int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname); /** * @brief Init file object in tfs. diff --git a/include/os/os.h b/include/os/os.h index b27fa84406b843f15ba99d52f7d9fde8580a0899..309a977ff6b7e6d500d93cba0ee487cc7befbd9e 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -27,7 +27,11 @@ extern "C" { #if !defined(WINDOWS) #include + +#if !defined(_ALPINE) #include +#endif + #include #include #include @@ -116,6 +120,7 @@ extern "C" { #include "osTimer.h" #include "osTimezone.h" #include "taoserror.h" +#include "tlog.h" #ifdef __cplusplus } diff --git a/include/os/osMemory.h b/include/os/osMemory.h index e6a4488287816f21ca9eeee76a3208ff6e6ff12d..3f57c72933115c5b34296f5730e81b656a53f50d 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -29,14 +29,17 @@ extern "C" { #define calloc CALLOC_FUNC_TAOS_FORBID #define realloc REALLOC_FUNC_TAOS_FORBID #define free FREE_FUNC_TAOS_FORBID +#define strdup STRDUP_FUNC_TAOS_FORBID #endif // ifndef ALLOW_FORBID_FUNC #endif // if !defined(WINDOWS) +int32_t taosMemoryDbgInit(); +int32_t taosMemoryDbgInitRestore(); void *taosMemoryMalloc(int64_t size); void *taosMemoryCalloc(int64_t num, int64_t size); void *taosMemoryRealloc(void *ptr, int64_t size); -void *taosMemoryStrDup(const char *ptr); +char *taosStrdup(const char *ptr); void taosMemoryFree(void *ptr); int64_t taosMemorySize(void *ptr); void taosPrintBackTrace(); diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index dbe4d6801e42fa04a9b9da0f2d3f864d0b75f94d..a8ccb67bfb1bb345150a2d7ee90d776e7b251aa3 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -60,6 +60,13 @@ void taosSetCoreDump(bool enable); #endif // WINDOWS +#if defined(_ALPINE) + +#define _UTSNAME_LENGTH 65 +#define _UTSNAME_MACHINE_LENGTH _UTSNAME_LENGTH + +#endif + typedef struct { char sysname[_UTSNAME_MACHINE_LENGTH]; char nodename[_UTSNAME_MACHINE_LENGTH]; diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 5154c56e4b5b48fb1a85e2dd620b3b3338902478..ba9f137b40edb2b4eacd2369004e1bd74bdd7361 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -16,6 +16,11 @@ #ifndef _TD_OS_SYSTEM_H_ #define _TD_OS_SYSTEM_H_ +#ifdef _ALPINE +#define UNW_LOCAL_ONLY +#include +#endif + #ifdef __cplusplus extern "C" { #endif @@ -29,46 +34,124 @@ extern "C" { #define tcgetattr TCGETATTR_FUNC_TAOS_FORBID #endif -typedef struct TdCmd* TdCmdPtr; +typedef struct TdCmd *TdCmdPtr; + +TdCmdPtr taosOpenCmd(const char *cmd); + +int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char *__restrict buf); + +int64_t taosGetLineCmd(TdCmdPtr pCmd, char **__restrict ptrBuf); + +int32_t taosEOFCmd(TdCmdPtr pCmd); -TdCmdPtr taosOpenCmd(const char* cmd); -int64_t taosGetsCmd(TdCmdPtr pCmd, int32_t maxSize, char* __restrict buf); -int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf); -int32_t taosEOFCmd(TdCmdPtr pCmd); -int64_t taosCloseCmd(TdCmdPtr* ppCmd); +int64_t taosCloseCmd(TdCmdPtr *ppCmd); -void* taosLoadDll(const char* filename); -void* taosLoadSym(void* handle, char* name); -void taosCloseDll(void* handle); +void *taosLoadDll(const char *filename); + +void *taosLoadSym(void *handle, char *name); + +void taosCloseDll(void *handle); int32_t taosSetConsoleEcho(bool on); -void taosSetTerminalMode(); + +void taosSetTerminalMode(); + int32_t taosGetOldTerminalMode(); -void taosResetTerminalMode(); + +void taosResetTerminalMode(); #define STACKSIZE 100 -#if !defined(WINDOWS) -#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \ - void* array[STACKSIZE]; \ - int32_t size = backtrace(array, STACKSIZE); \ - char** strings = backtrace_symbols(array, size); \ - int32_t offset = 0; \ - if (strings != NULL) { \ - offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \ - for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ - offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, strings[i]); \ - } \ - } \ - \ - taosMemoryFree(strings); \ -} +#if defined(_ALPINE) +#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \ + { \ + unw_cursor_t cursor; \ + unw_context_t context; \ + \ + unw_getcontext(&context); \ + unw_init_local(&cursor, &context); \ + \ + char *array[STACKSIZE]; \ + int32_t size = 0; \ + int32_t ignores = ignoreNum; \ + int32_t offset = 0; \ + while (unw_step(&cursor) > 0 && size < STACKSIZE) { \ + unw_word_t offset, pc; \ + char fname[64]; \ + unw_get_reg(&cursor, UNW_REG_IP, &pc); \ + fname[0] = '\0'; \ + (void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \ + size += 1; \ + array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \ + snprintf(array[size], STACKSIZE, "0x%lx : (%s+0x%lx) [0x%lx]\n", (long)pc, fname, (long)offset, (long)pc); \ + } \ + if (ignoreNum < size && size > 0) { \ + offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \ + for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ + offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \ + array[i]); \ + } \ + } \ + for (int i = 0; i < size; i++) { \ + taosMemoryFree(array[i]); \ + } \ + } + +#define taosPrintTrace(flags, level, dflag, ignoreNum) \ + { \ + unw_cursor_t cursor; \ + unw_context_t context; \ + \ + unw_getcontext(&context); \ + unw_init_local(&cursor, &context); \ + \ + char *array[STACKSIZE]; \ + int32_t size = 0; \ + while (unw_step(&cursor) > 0 && size < STACKSIZE) { \ + unw_word_t offset, pc; \ + char fname[64]; \ + unw_get_reg(&cursor, UNW_REG_IP, &pc); \ + fname[0] = '\0'; \ + (void)unw_get_proc_name(&cursor, fname, sizeof(fname), &offset); \ + size += 1; \ + array[size] = (char *)taosMemoryMalloc(sizeof(char) * STACKSIZE + 1); \ + snprintf(array[size], STACKSIZE, "frame:%d, 0x%lx : (%s+0x%lx) [0x%lx]\n", size, (long)pc, fname, (long)offset, \ + (long)pc); \ + } \ + if (ignoreNum < size && size > 0) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames", (ignoreNum > 0) ? size - ignoreNum : size); \ + for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ + taosPrintLog(flags, level, dflag, "frame:%d, %s", (ignoreNum > 0) ? i - ignoreNum : i, array[i]); \ + } \ + } \ + for (int i = 0; i < size; i++) { \ + taosMemoryFree(array[i]); \ + } \ + } + +#elif !defined(WINDOWS) +#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \ + { \ + void *array[STACKSIZE]; \ + int32_t size = backtrace(array, STACKSIZE); \ + char **strings = backtrace_symbols(array, size); \ + int32_t offset = 0; \ + if (strings != NULL) { \ + offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? size - ignoreNum : size); \ + for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ + offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%d, %s\n", (ignoreNum > 0) ? i - ignoreNum : i, \ + strings[i]); \ + } \ + } \ + \ + taosMemoryFree(strings); \ + } #define taosPrintTrace(flags, level, dflag, ignoreNum) \ { \ - void* array[STACKSIZE]; \ - int32_t size = backtrace(array, STACKSIZE); \ - char** strings = backtrace_symbols(array, size); \ + void *array[STACKSIZE]; \ + int32_t size = backtrace(array, STACKSIZE); \ + char **strings = backtrace_symbols(array, size); \ if (strings != NULL) { \ taosPrintLog(flags, level, dflag, "obtained %d stack frames", (ignoreNum > 0) ? size - ignoreNum : size); \ for (int32_t i = (ignoreNum > 0) ? ignoreNum : 0; i < size; i++) { \ @@ -80,65 +163,70 @@ void taosResetTerminalMode(); } #else -#include #include +#include -#define taosLogTraceToBuf(buf, bufSize, ignoreNum) { \ - unsigned int i; \ - void* stack[STACKSIZE]; \ - unsigned short frames; \ - SYMBOL_INFO* symbol; \ - HANDLE process; \ - int32_t offset = 0; \ - \ - process = GetCurrentProcess(); \ - \ - SymInitialize(process, NULL, TRUE); \ - \ - frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ - symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ - if (symbol != NULL) { \ - symbol->MaxNameLen = 255; \ - symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ - \ - if (frames > 0) { \ - offset = snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \ - for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ - SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ - offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \ - } \ - } \ - free(symbol); \ - } \ +#define taosLogTraceToBuf(buf, bufSize, ignoreNum) \ + { \ + unsigned int i; \ + void *stack[STACKSIZE]; \ + unsigned short frames; \ + SYMBOL_INFO *symbol; \ + HANDLE process; \ + int32_t offset = 0; \ + \ + process = GetCurrentProcess(); \ + \ + SymInitialize(process, NULL, TRUE); \ + \ + frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ + symbol = (SYMBOL_INFO *)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ + if (symbol != NULL) { \ + symbol->MaxNameLen = 255; \ + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ + \ + if (frames > 0) { \ + offset = \ + snprintf(buf, bufSize - 1, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \ + for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ + SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ + offset += snprintf(buf + offset, bufSize - 1 - offset, "frame:%i, %s - 0x%0X\n", \ + (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \ + } \ + } \ + free(symbol); \ + } \ } -#define taosPrintTrace(flags, level, dflag, ignoreNum) \ +#define taosPrintTrace(flags, level, dflag, ignoreNum) \ { \ - unsigned int i; \ - void* stack[STACKSIZE]; \ - unsigned short frames; \ - SYMBOL_INFO* symbol; \ - HANDLE process; \ - \ - process = GetCurrentProcess(); \ - \ - SymInitialize(process, NULL, TRUE); \ - \ - frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ - symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ - if (symbol != NULL) { \ - symbol->MaxNameLen = 255; \ - symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ - \ - if (frames > 0) { \ - taosPrintLog(flags, level, dflag, "obtained %d stack frames\n", (ignoreNum > 0) ? frames - ignoreNum : frames); \ - for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ - SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ - taosPrintLog(flags, level, dflag, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, symbol->Name, symbol->Address); \ - } \ - } \ - free(symbol); \ - } \ + unsigned int i; \ + void *stack[STACKSIZE]; \ + unsigned short frames; \ + SYMBOL_INFO *symbol; \ + HANDLE process; \ + \ + process = GetCurrentProcess(); \ + \ + SymInitialize(process, NULL, TRUE); \ + \ + frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \ + symbol = (SYMBOL_INFO *)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \ + if (symbol != NULL) { \ + symbol->MaxNameLen = 255; \ + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \ + \ + if (frames > 0) { \ + taosPrintLog(flags, level, dflag, "obtained %d stack frames\n", \ + (ignoreNum > 0) ? frames - ignoreNum : frames); \ + for (i = (ignoreNum > 0) ? ignoreNum : 0; i < frames; i++) { \ + SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \ + taosPrintLog(flags, level, dflag, "frame:%i, %s - 0x%0X\n", (ignoreNum > 0) ? i - ignoreNum : i, \ + symbol->Name, symbol->Address); \ + } \ + } \ + free(symbol); \ + } \ } #endif diff --git a/include/os/osThread.h b/include/os/osThread.h index 916d463733373c2512ab77a7e6931cede98966af..aa0dc066c60bb7e218f249bb06505c43c7d2845c 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,7 +22,7 @@ extern "C" { #endif -#ifndef WINDOWS +#if !defined(WINDOWS) && !defined(_ALPINE) #ifndef __USE_XOPEN2K #define TD_USE_SPINLOCK_AS_MUTEX typedef pthread_mutex_t pthread_spinlock_t; @@ -100,7 +100,11 @@ typedef pthread_key_t TdThreadKey; #define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID #define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID #define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID + +#if !defined(_ALPINE) #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID +#endif + #define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID #define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID #define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID diff --git a/include/util/tRealloc.h b/include/util/tRealloc.h index f3593d5818cd626c7268ec1b721bd961fba3a845..3229c53039f6dc857dd7e0b2d4774e6875ef8c16 100644 --- a/include/util/tRealloc.h +++ b/include/util/tRealloc.h @@ -52,11 +52,13 @@ _exit: return code; } -static FORCE_INLINE void tFree(uint8_t *pBuf) { - if (pBuf) { - taosMemoryFree(pBuf - sizeof(int64_t)); - } -} +#define tFree(BUF) \ + do { \ + if (BUF) { \ + taosMemoryFree((uint8_t *)(BUF) - sizeof(int64_t)); \ + (BUF) = NULL; \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 75bdf81a2750f2fc6c8a9fad42c568299197b666..75860a4b1e3524c40b15793bf11056025679881e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -119,6 +119,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) // #define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) // +#define TSDB_CODE_IVLD_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) // //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -163,6 +164,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X022D) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) +#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) +#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) // mnode-common // #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x @@ -255,6 +258,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) + // mnode-func #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) // #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) // 2.x @@ -267,6 +271,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) + + // mnode-db #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) // @@ -361,12 +367,18 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4) #define TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB TAOS_DEF_ERROR_CODE(0, 0x03F5) #define TSDB_CODE_MND_TOO_MANY_STREAMS TAOS_DEF_ERROR_CODE(0, 0x03F6) +#define TSDB_CODE_MND_INVALID_TARGET_TABLE TAOS_DEF_ERROR_CODE(0, 0x03F7) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) #define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481) #define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482) +// mnode-tag-indxe + +#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0483) +#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0484) + // dnode // #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x // #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) // 2.x @@ -419,6 +431,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_NO_AVAIL_BUFPOOL TAOS_DEF_ERROR_CODE(0, 0x0528) #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) +#define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -705,6 +718,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) #define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) #define TSDB_CODE_SML_NOT_SAME_TYPE TAOS_DEF_ERROR_CODE(0, 0x3004) +#define TSDB_CODE_SML_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x3005) //tsma #define TSDB_CODE_TSMA_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x3100) @@ -720,7 +734,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151) #define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152) -// #define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153) +#define TSDB_CODE_RSMA_FS_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3153) #define TSDB_CODE_RSMA_REMOVE_EXISTS TAOS_DEF_ERROR_CODE(0, 0x3154) #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x3155) #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) @@ -728,6 +742,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_REGEX_MATCH TAOS_DEF_ERROR_CODE(0, 0x3158) #define TSDB_CODE_RSMA_STREAM_STATE_OPEN TAOS_DEF_ERROR_CODE(0, 0x3159) #define TSDB_CODE_RSMA_STREAM_STATE_COMMIT TAOS_DEF_ERROR_CODE(0, 0x3160) +#define TSDB_CODE_RSMA_FS_REF TAOS_DEF_ERROR_CODE(0, 0x3161) +#define TSDB_CODE_RSMA_FS_SYNC TAOS_DEF_ERROR_CODE(0, 0x3162) +#define TSDB_CODE_RSMA_FS_UPDATE TAOS_DEF_ERROR_CODE(0, 0x3163) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/include/util/tarray.h b/include/util/tarray.h index fe758f06c7c7489bad93fe65ec127c58ed7f3010..a8510e4bc8295e7e40092ea5ae687717bda20676 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -22,19 +22,6 @@ extern "C" { #endif -#if 0 -#define TARRAY(TYPE) \ - struct { \ - int32_t tarray_size_; \ - int32_t tarray_neles_; \ - struct TYPE* td_array_data_; \ - } - -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ -#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) -#endif - #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) #define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) @@ -46,6 +33,9 @@ typedef struct SArray { void* pData; } SArray; +#define TARRAY_SIZE(array) ((array)->size) +#define TARRAY_DATA(array) ((array)->pData) + /** * * @param size @@ -187,6 +177,13 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt); */ void taosArrayRemove(SArray* pArray, size_t index); +/** + * remove batch entry from the given index + * @param pArray + * @param index + */ +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp); + /** * copy the whole array from source to destination * @param pDst diff --git a/include/util/tdef.h b/include/util/tdef.h index aeb8d08936d4de875fc2c5754d562dc57cb5333c..85c89744ed88fdf27adb2795baf70028238906b6 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -104,6 +104,7 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" +#define TSDB_INDEX_TYPE_NORMAL "NORMAL" #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 @@ -190,10 +191,11 @@ typedef enum ELogicConditionType { #define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns #define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_CGROUP_LEN 193 // it is a null-terminated string -#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_CGROUP_LEN 193 // it is a null-terminated string +#define TSDB_USER_CGROUP_LEN (TSDB_USER_LEN + TSDB_CGROUP_LEN) // it is a null-terminated string +#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) diff --git a/include/util/tencode.h b/include/util/tencode.h index a6dd58297e8c1dba644d86eb5145b273406fbf9e..ff97a205073822be2c98e39469066a0470794a51 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -116,6 +116,7 @@ static int32_t tEncodeI64v(SEncoder* pCoder, int64_t val); static int32_t tEncodeFloat(SEncoder* pCoder, float val); static int32_t tEncodeDouble(SEncoder* pCoder, double val); static int32_t tEncodeBinary(SEncoder* pCoder, const uint8_t* val, uint32_t len); +static int32_t tEncodeBinaryEx(SEncoder* pCoder, const uint8_t* val, uint32_t len); static int32_t tEncodeCStrWithLen(SEncoder* pCoder, const char* val, uint32_t len); static int32_t tEncodeCStr(SEncoder* pCoder, const char* val); diff --git a/include/util/tlog.h b/include/util/tlog.h index 4311719ca575e2ccc69d3281d8f49d1e006d24aa..0071b3d32cd72be79ab5adf8bdba20b92c18f4f7 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -89,20 +89,20 @@ bool taosAssertRelease(bool condition); // Disable all asserts that may compromise the performance. #if defined DISABLE_ASSERT #define ASSERT(condition) -#define ASSERTS(condition, ...) (0) +#define ASSERTS(condition, ...) (0) #else -#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__) +#define ASSERTS(condition, ...) ((condition) ? false : taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__)) #ifdef NDEBUG #define ASSERT(condition) taosAssertRelease(condition) #else -#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided") +#define ASSERT(condition) ASSERTS(condition, "assert info not provided") #endif #endif -void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo); -void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd); -void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); -int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime); +void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo); +void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd); +void taosReleaseCrashLogFile(TdFilePtr pFile, bool truncateFile); +int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime); // clang-format off #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} @@ -116,7 +116,7 @@ int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } // clang-format on -//#define BUF_PAGE_DEBUG +// #define BUF_PAGE_DEBUG #ifdef __cplusplus } #endif diff --git a/include/util/trbtree.h b/include/util/trbtree.h index 7c93c8dd8d65f2c4947019f370c0933e6ad81ee0..e2264194401bee65a22f0301f0339cb76a8d7356 100644 --- a/include/util/trbtree.h +++ b/include/util/trbtree.h @@ -33,9 +33,12 @@ typedef int32_t (*tRBTreeCmprFn)(const SRBTreeNode *, const SRBTreeNode *); #define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max) void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); +void tRBTreeClear(SRBTree *pTree); SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeDropMin(SRBTree *pTree); +SRBTreeNode *tRBTreeDropMax(SRBTree *pTree); SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode); // SRBTreeIter ============================================= diff --git a/packaging/docker/README.md b/packaging/docker/README.md index 763ab73724587eb4dc231eb399a60937eaba6dca..4509a7a1a9ae521906e19e018a3f2037a7bb5a5f 100644 --- a/packaging/docker/README.md +++ b/packaging/docker/README.md @@ -18,65 +18,58 @@ TDengine is an open-sourced big data platform under [GNU AGPL v3.0](http://www.g ## How to use this image -### Start a TDengine instance with RESTful API exposed +### Starting TDengine -Simply, you can use `docker run` to start a TDengine instance and connect it with restful connectors(eg. [JDBC-RESTful](https://www.taosdata.com/cn/documentation/connector/java)). +The TDengine image starts with the HTTP service activated by default, using the following command: -```bash +```shell docker run -d --name tdengine -p 6041:6041 tdengine/tdengine ``` -This command starts a docker container by name `tdengine` with TDengine server running, and maps the container's HTTP port 6041 to the host's port 6041. If you have `curl` in your host, you can list the databases by the command: +The above command starts a container named "tdengine" and maps the HTTP service port 6041 to the host port 6041. You can verify that the HTTP service provided in this container is available using the following command. -```bash +```shell curl -u root:taosdata -d "show databases" localhost:6041/rest/sql ``` -You can execute the `taos` shell command in the container: +The TDengine client taos can be executed in this container to access TDengine using the following command. -```bash +```shell $ docker exec -it tdengine taos -Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. - taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | -Query OK, 1 row(s) in set (0.002843s) + name | +================================= + information_schema | + performance_schema | +Query OK, 2 row(s) in set (0.002843s) ``` -Since TDengine use container hostname to establish connections, it's a bit more complex to use TDengine CLI and native connectors(such as JDBC-JNI) with TDengine container instance. This is the recommended way to expose ports and use TDengine with docker in simple cases. If you want to use TDengine CLI or taosc/connectors smoothly outside the `tdengine` container, see next use cases that match you need. +The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios. -### Start with host network +### Start TDengine on the host network -```bash +```shell docker run -d --name tdengine --network host tdengine/tdengine ``` -Starts container with `host` network will use host's hostname as fqdn instead of container id. It's much like starting natively with `systemd` in host. After installing the client, you can use `taos` shell as normal in host path. +The above command starts TDengine on the host network and uses the host's FQDN to establish a connection instead of the container's hostname. It is the equivalent of using `systemctl` to start TDengine on the host. If the TDengine client is already installed on the host, you can access it directly with the following command. -```bash +```shell $ taos -Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. - taos> show dnodes; id | end_point | vnodes | cores | status | role | create_time | offline reason | ====================================================================================================================================== - 1 | host:6030 | 1 | 8 | ready | any | 2022-01-17 22:10:32.619 | | + 1 | myhost:6030 | 1 | 8 | ready | any | 2022-01-17 22:10:32.619 | | Query OK, 1 row(s) in set (0.003233s) ``` -### Start with exposed ports and specified hostname - -Set the fqdn explicitly will help you to use in other environment or applications. We provide environment variable `TAOS_FQDN` or `fqdn` config option to explicitly set the hostname used by TDengine container instance(s). +### Start TDengine with the specified hostname and port -Use `TAOS_FQDN` variable within `docker run` command: +The `TAOS_FQDN` environment variable or the `fqdn` configuration item in `taos.cfg` allows TDengine to establish a connection at the specified hostname. This approach provides greater flexibility for deployment. -```bash +```shell docker run -d \ --name tdengine \ -e TAOS_FQDN=tdengine \ @@ -85,79 +78,58 @@ docker run -d \ tdengine/tdengine ``` -This command starts a docker container with TDengine server running and maps the container's TCP ports from 6030 to 6049 to the host's ports from 6030 to 6049 with TCP protocol and UDP ports range 6030-6039 to the host's UDP ports 6030-6039. If the host is already running TDengine server and occupying the same port(s), you need to map the container's port to a different unused port segment. (Please see TDengine 2.0 Port Description for details). In order to support TDengine clients accessing TDengine server services, both TCP and UDP ports need to be exposed by default(unless `rpcForceTcp` is set to `1`). +The above command starts a TDengine service in the container, which listens to the hostname tdengine, and maps the container's port segment 6030 to 6049 to the host's port segment 6030 to 6049 (both TCP and UDP ports need to be mapped). If the port segment is already occupied on the host, you can modify the above command to specify a free port segment on the host. If `rpcForceTcp` is set to `1`, you can map only the TCP protocol. -If you want to use TDengine CLI or native connectors([JDBC-JNI](https://www.taosdata.com/cn/documentation/connector/java), or [driver-go](https://github.com/taosdata/driver-go)), you need to make sure the `TAOS_FQDN` is resolvable at `/etc/hosts` or with custom DNS service. +Next, ensure the hostname "tdengine" is resolvable in `/etc/hosts`. -If you set the `TAOS_FQDN` to host's hostname, it will works as using `hosts` network like previous use case. Otherwise, like in `-e TAOS_FQDN=tdengine`, you can add the hostname record `tdengine` into `/etc/hosts` (use `127.0.0.1` here in host path, if use TDengine client/application in other hosts, you should set the right ip to the host eg. `192.168.10.1`(check the real ip in host with `hostname -i` or `ip route list default`) to make the TDengine endpoint resolvable): - -```bash +```shell echo 127.0.0.1 tdengine |sudo tee -a /etc/hosts ``` -Then you can use `taos` with the host `tdengine`: +Finally, the TDengine service can be accessed from the TDengine CLI or any connector with "tdengine" as the server address. -```bash -taos -h tdengine +```shell +taos -h tdengine -P 6030 ``` -Or develop/test applications with native connectors. As in python: +If set `TAOS_FQDN` to the same hostname, the effect is the same as "Start TDengine on host network". -```python -import taos; -conn = taos.connect(host = "tdengine") -res = conn.query("show databases") -for row in res.fetch_all_into_dict(): - print(row) -``` +### Start TDengine on the specified network -See the results: - -```bash -Python 3.8.10 (default, Nov 26 2021, 20:14:08) -[GCC 9.3.0] on linux -Type "help", "copyright", "credits" or "license" for more information. ->>> import taos; ->>> conn = taos.connect(host = "tdengine") ->>> res = conn.query("show databases") ->>> for row in res.fetch_all_into_dict(): -... print(row) -... -{'name': 'log', 'created_time': datetime.datetime(2022, 1, 17, 22, 56, 2, 490000), 'ntables': 11, 'vgroups': 1, 'replica': 1, 'quorum': 1, 'days': 10, 'keep': '30', 'cache(MB)': 1, 'blocks': 3, 'minrows': 100, 'maxrows': 4096, 'wallevel': 1, 'fsync': 3000, 'comp': 2, 'cachelast': 0, 'precision': 'us', 'update': 0, 'status': 'ready'} -``` +You can also start TDengine on a specific network. Perform the following steps: -### Start with specific network +1. First, create a docker network named `td-net` -Alternatively, you can use TDengine natively by using specific network. + ```shell + docker network create td-net + ``` -First, create network for TDengine server and client/application. +2. Start TDengine -```bash -docker network create td-net -``` + Start the TDengine service on the `td-net` network with the following command: -Start TDengine instance with service name as fqdn (explicitly set with `TAOS_FQDN`): + ```shell + docker run -d --name tdengine --network td-net \ + -e TAOS_FQDN=tdengine \ + tdengine/tdengine + ``` -```bash -docker run -d --name tdengine --network td-net \ - -e TAOS_FQDN=tdengine \ - tdengine/tdengine -``` +3. Start the TDengine client in another container on the same network -Start TDengine client in another container with the specific network: + ```shell + docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos + # or + #docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine + ``` -```bash -docker run --rm -it --network td-net -e TAOS_FIRST_EP=tdengine tdengine/tdengine taos -# or -docker run --rm -it --network td-net -e tdengine/tdengine taos -h tdengine -``` +### Launching a client application in a container -When you build your application with docker, you should add the TDengine client in the dockerfile, as based on `ubuntu:20.04` image, install the client like this: +If you want to start your application in a container, you need to add the corresponding dependencies on TDengine to the image as well, e.g. -```dockerfile +```docker FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget -ENV TDENGINE_VERSION=2.4.0.0 +ENV TDENGINE_VERSION=3.0.0.0 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ @@ -169,10 +141,7 @@ RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENG #CMD ["app"] ``` -Here is an Go example app: - - - +Here is an example GO program: ```go /* @@ -181,19 +150,19 @@ Here is an Go example app: package main import ( - "database/sql" - "flag" - "fmt" - "time" + "database/sql" + "flag" + "fmt" + "time" - _ "github.com/taosdata/driver-go/v2/taosSql" + _ "github.com/taosdata/driver-go/v3/taosSql" ) type config struct { - hostName string - serverPort string - user string - password string + hostName string + serverPort string + user string + password string } var configPara config @@ -201,70 +170,67 @@ var taosDriverName = "taosSql" var url string func init() { - flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.") - flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.") - flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") - flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") - flag.Parse() + flag.StringVar(&configPara.hostName, "h", "", "The host to connect to TDengine server.") + flag.StringVar(&configPara.serverPort, "p", "", "The TCP/IP port number to use for the connection to TDengine server.") + flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") + flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") + flag.Parse() } func printAllArgs() { - fmt.Printf("============= args parse result: =============\n") - fmt.Printf("hostName: %v\n", configPara.hostName) - fmt.Printf("serverPort: %v\n", configPara.serverPort) - fmt.Printf("usr: %v\n", configPara.user) - fmt.Printf("password: %v\n", configPara.password) - fmt.Printf("================================================\n") + fmt.Printf("============= args parse result: =============\n") + fmt.Printf("hostName: %v\n", configPara.hostName) + fmt.Printf("serverPort: %v\n", configPara.serverPort) + fmt.Printf("usr: %v\n", configPara.user) + fmt.Printf("password: %v\n", configPara.password) + fmt.Printf("================================================\n") } func main() { - printAllArgs() - - url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/" - - taos, err := sql.Open(taosDriverName, url) - checkErr(err, "open database error") - defer taos.Close() - - taos.Exec("create database if not exists test") - taos.Exec("use test") - taos.Exec("create table if not exists tb1 (ts timestamp, a int)") - _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") - checkErr(err, "failed to insert") - rows, err := taos.Query("select * from tb1") - checkErr(err, "failed to select") - - defer rows.Close() - for rows.Next() { - var r struct { - ts time.Time - a int - } - err := rows.Scan(&r.ts, &r.a) - if err != nil { - fmt.Println("scan error:\n", err) - return - } - fmt.Println(r.ts, r.a) - } + printAllArgs() + + url = "root:taosdata@/tcp(" + configPara.hostName + ":" + configPara.serverPort + ")/" + + taos, err := sql.Open(taosDriverName, url) + checkErr(err, "open database error") + defer taos.Close() + + taos.Exec("create database if not exists test") + taos.Exec("use test") + taos.Exec("create table if not exists tb1 (ts timestamp, a int)") + _, err = taos.Exec("insert into tb1 values(now, 0)(now+1s,1)(now+2s,2)(now+3s,3)") + checkErr(err, "failed to insert") + rows, err := taos.Query("select * from tb1") + checkErr(err, "failed to select") + + defer rows.Close() + for rows.Next() { + var r struct { + ts time.Time + a int + } + err := rows.Scan(&r.ts, &r.a) + if err != nil { + fmt.Println("scan error:\n", err) + return + } + fmt.Println(r.ts, r.a) + } } func checkErr(err error, prompt string) { - if err != nil { - fmt.Println("ERROR: %s\n", prompt) - panic(err) - } + if err != nil { + fmt.Println("ERROR: %s\n", prompt) + panic(err) + } } ``` - - - -Full version of dockerfile could be: +Here is the full Dockerfile: -```dockerfile +```docker FROM golang:1.17.6-buster as builder -ENV TDENGINE_VERSION=2.4.0.0 +ENV TDENGINE_VERSION=3.0.0.0 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ @@ -274,11 +240,13 @@ RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENG WORKDIR /usr/src/app/ ENV GOPROXY="https://goproxy.io,direct" COPY ./main.go ./go.mod ./go.sum /usr/src/app/ -RUN go env && go mod tidy && go build +RUN go env +RUN go mod tidy +RUN go build FROM ubuntu:20.04 RUN apt-get update && apt-get install -y wget -ENV TDENGINE_VERSION=2.4.0.0 +ENV TDENGINE_VERSION=3.0.0.0 RUN wget -c https://www.taosdata.com/assets-download/3.0/TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && tar xvf TDengine-client-${TDENGINE_VERSION}-Linux-x64.tar.gz \ && cd TDengine-client-${TDENGINE_VERSION} \ @@ -291,9 +259,9 @@ COPY --from=builder /usr/src/app/app /usr/bin/ CMD ["app"] ``` -Suppose you have `main.go`, `go.mod` `go.sum`, `app.dockerfile`, build the app and run it with network `td-net`: +Now that we have `main.go`, `go.mod`, `go.sum`, `app.dockerfile`, we can build the application and start it on the `td-net` network. -```bash +```shell $ docker build -t app -f app.dockerfile $ docker run --rm --network td-net app -h tdengine -p 6030 ============= args parse result: ============= @@ -316,26 +284,18 @@ password: taosdata 2022-01-18 01:43:51.029 +0000 UTC 3 ``` -Now you must be much familiar with developing and testing with TDengine, let's see some more complex cases. +### Start the TDengine cluster with docker-compose -### Start with docker-compose with multiple nodes(instances) +1. The following docker-compose file starts a TDengine cluster with three nodes. -Start a 2-replicas-2-mnodes-2-dnodes-1-arbitrator TDengine cluster with `docker-compose` is quite simple. Save the file as `docker-compose.yml`: - -```yaml +```yml version: "3" services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator td-1: image: tdengine/tdengine:$VERSION environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 volumes: - taosdata-td1:/var/lib/taos/ - taoslog-td1:/var/log/taos/ @@ -344,101 +304,95 @@ services: environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 volumes: - taosdata-td2:/var/lib/taos/ - taoslog-td2:/var/log/taos/ + td-3: + image: tdengine/tdengine:$VERSION + environment: + TAOS_FQDN: "td-3" + TAOS_FIRST_EP: "td-1" + volumes: + - taosdata-td3:/var/lib/taos/ + - taoslog-td3:/var/log/taos/ volumes: taosdata-td1: taoslog-td1: taosdata-td2: taoslog-td2: + taosdata-td3: + taoslog-td3: ``` -You may notice that: - -- We use `VERSION` environment variable to set `tdengine` image tag version once. -- **`TAOS_FIRST_EP`** **MUST** be set to join the newly created instances into an existing TDengine cluster. If you want more instances, use `TAOS_SECOND_EP` in case of HA(High Availability) concerns. -- `TAOS_NUM_OF_MNODES` is for setting number of mnodes for the cluster. -- `TAOS_REPLICA` set the default database replicas, `2` means there're one master and one slave copy of data. The `replica` option should be `1 <= replica <= 3`, and not greater than dnodes number. -- `TAOS_ARBITRATOR` set the arbitrator entrypoint of the cluster for failover/election stuff. It's better to use arbitrator in a two nodes cluster. -- The way to start an arbitrator service is as easy as abc: just add command name `tarbitrator`(which is the binary name of arbitrator daemon) in docker-compose service option: `command: tarbitrator`, and everything is ok now. - -Now run `docker-compose up -d` with version specified: - -```bash -$ VERSION=2.4.0.0 docker-compose up -d -Creating network "test_default" with the default driver -Creating volume "test_taosdata-td1" with default driver -Creating volume "test_taoslog-td1" with default driver -Creating volume "test_taosdata-td2" with default driver -Creating volume "test_taoslog-td2" with default driver -Creating test_td-1_1 ... done -Creating test_arbitrator_1 ... done -Creating test_td-2_1 ... done -``` +:::note -Check the status: +- The `VERSION` environment variable is used to set the tdengine image tag +- `TAOS_FIRST_EP` must be set on the newly created instance so that it can join the TDengine cluster; if there is a high availability requirement, `TAOS_SECOND_EP` needs to be used at the same time + ::: -```bash -$ docker-compose ps - Name Command State Ports ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -test_arbitrator_1 /usr/bin/entrypoint.sh tar ... Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp -test_td-1_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp -test_td-2_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp -``` +2. Start the cluster -Check dnodes with TDengine CLI: + ```shell + $ VERSION=3.0.0.0 docker-compose up -d + Creating network "test_default" with the default driver + Creating volume "test_taosdata-td1" with default driver + Creating volume "test_taoslog-td1" with default driver + Creating volume "test_taosdata-td2" with default driver + Creating volume "test_taoslog-td2" with default driver + Creating test_td-1_1 ... done + Creating test_arbitrator_1 ... done + Creating test_td-2_1 ... done + ``` -```bash -$ docker-compose exec td-1 taos -s "show dnodes" +3. Check the status of each node + + ```shell + $ docker-compose ps + Name Command State Ports + --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + test_arbitrator_1 /usr/bin/entrypoint.sh tar ... Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp + test_td-1_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp + test_td-2_1 /usr/bin/entrypoint.sh taosd Up 6030/tcp, 6031/tcp, 6032/tcp, 6033/tcp, 6034/tcp, 6035/tcp, 6036/tcp, 6037/tcp, 6038/tcp, 6039/tcp, 6040/tcp, 6041/tcp, 6042/tcp + ``` -Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +4. Show dnodes via TDengine CLI + +```shell +$ docker-compose exec td-1 taos -s "show dnodes" taos> show dnodes - id | end_point | vnodes | cores | status | role | create_time | offline reason | + id | endpoint | vnodes | support_vnodes | status | create_time | note | ====================================================================================================================================== - 1 | td-1:6030 | 1 | 8 | ready | any | 2022-01-18 02:47:42.871 | | - 2 | td-2:6030 | 0 | 8 | ready | any | 2022-01-18 02:47:43.518 | | - 0 | arbitrator:6042 | 0 | 0 | ready | arb | 2022-01-18 02:47:43.633 | - | -Query OK, 3 row(s) in set (0.000811s) -``` + 1 | td-1:6030 | 0 | 32 | ready | 2022-08-19 07:57:29.971 | | + 2 | td-2:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.415 | | + 3 | td-3:6030 | 0 | 32 | ready | 2022-08-19 07:57:31.417 | | +Query OK, 3 rows in database (0.021262s) -### Start a TDengine cluster with scaled taosadapter service +``` -In previous use case, you could see the way to start other services built with TDengine(`taosd` as the default command). There's another important service you should know: +## taosAdapter -> **taosAdapter** is a TDengine’s companion tool and is a bridge/adapter between TDengine cluster and application. It provides an easy-to-use and efficient way to ingest data from data collections agents(like Telegraf, StatsD, CollectD) directly. It also provides InfluxDB/OpenTSDB compatible data ingestion interface to allow InfluxDB/OpenTSDB applications to immigrate to TDengine seamlessly. +1. taosAdapter is enabled by default in the TDengine container. If you want to disable it, specify the environment variable `TAOS_DISABLE_ADAPTER=true` at startup -`taosadapter` is running inside `tdengine` image by default, you can disable it by `TAOS_DISABLE_ADAPTER=true`. Running `taosadapter` in a separate container is like how `arbitrator` does: +2. At the same time, for flexible deployment, taosAdapter can be started in a separate container -```yaml -services: - # ... - adapter: - image: tdengine/tdengine:$VERSION - command: taosadapter -``` + ```docker + services: + # ... + adapter: + image: tdengine/tdengine:$VERSION + command: taosadapter + ``` -`taosadapter` could be scaled with docker-compose, so that you can manage the `taosadapter` nodes easily. Here is an example shows 4-`taosadapter` instances in a TDengine cluster(much like previous use cases): + Suppose you want to deploy multiple taosAdapters to improve throughput and provide high availability. In that case, the recommended configuration method uses a reverse proxy such as Nginx to offer a unified access entry. For specific configuration methods, please refer to the official documentation of Nginx. Here is an example: -```yaml +```yml version: "3" networks: inter: - api: services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - networks: - - inter td-1: image: tdengine/tdengine:$VERSION networks: @@ -446,9 +400,6 @@ services: environment: TAOS_FQDN: "td-1" TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 volumes: - taosdata-td1:/var/lib/taos/ - taoslog-td1:/var/log/taos/ @@ -459,15 +410,12 @@ services: environment: TAOS_FQDN: "td-2" TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 volumes: - taosdata-td2:/var/lib/taos/ - taoslog-td2:/var/log/taos/ adapter: image: tdengine/tdengine:$VERSION - command: taosadapter + entrypoint: "taosadapter" networks: - inter environment: @@ -481,7 +429,6 @@ services: - adapter networks: - inter - - api ports: - 6041:6041 - 6044:6044/udp @@ -504,100 +451,14 @@ volumes: taoslog-td2: ``` -Start the cluster: - -```bash -$ VERSION=2.4.0.0 docker-compose up -d -Creating network "docker_inter" with the default driver -Creating network "docker_api" with the default driver -Creating volume "docker_taosdata-td1" with default driver -Creating volume "docker_taoslog-td1" with default driver -Creating volume "docker_taosdata-td2" with default driver -Creating volume "docker_taoslog-td2" with default driver -Creating docker_td-2_1 ... done -Creating docker_arbitrator_1 ... done -Creating docker_td-1_1 ... done -Creating docker_adapter_1 ... done -Creating docker_adapter_2 ... done -Creating docker_adapter_3 ... done -``` - -It will start a TDengine cluster with two dnodes and four taosadapter instances, expose ports 6041/tcp and 6044/udp to host. - -`6041` is the RESTful API endpoint port, you can verify that the RESTful interface taosAdapter provides working using the `curl` command. - -```bash -$ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql -{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","cachelast","precision","update","status"],"column_meta":[["name",8,32],["created_time",9,8],["ntables",4,4],["vgroups",4,4],["replica",3,2],["quorum",3,2],["days",3,2],["keep",8,24],["cache(MB)",4,4],["blocks",4,4],["minrows",4,4],["maxrows",4,4],["wallevel",2,1],["fsync",4,4],["comp",2,1],["cachelast",2,1],["precision",8,3],["update",2,1],["status",8,10]],"data":[["log","2022-01-18 04:37:42.902",16,1,1,1,10,"30",1,3,100,4096,1,3000,2,0,"us",0,"ready"]],"rows":1} -``` - -If you run curl in batch(here we use [hyperfine](https://github.com/sharkdp/hyperfine) - a command-line benchmarking tool), the requests are balanced into 4 adapter instances. - -```bash -hyperfine -m10 'curl -u root:taosdata localhost:6041/rest/sql -d "describe log.log"' -``` - -View the logs with `docker-compose logs`: - -```bash -$ docker-compose logs adapter -# some logs skipped -adapter_2 | 01/18 04:57:44.616529 00000039 TAOS_ADAPTER info "| 200 | 162.185µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 -adapter_1 | 01/18 04:57:44.627695 00000039 TAOS_ADAPTER info "| 200 | 145.485µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=17 -adapter_3 | 01/18 04:57:44.639165 00000040 TAOS_ADAPTER info "| 200 | 146.913µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web -adapter_4 | 01/18 04:57:44.650829 00000039 TAOS_ADAPTER info "| 200 | 153.201µs | 172.21.0.9 | POST | /rest/sql " sessionID=17 model=web -adapter_2 | 01/18 04:57:44.662422 00000039 TAOS_ADAPTER info "| 200 | 211.393µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 -adapter_1 | 01/18 04:57:44.673426 00000039 TAOS_ADAPTER info "| 200 | 154.714µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 -adapter_3 | 01/18 04:57:44.684788 00000040 TAOS_ADAPTER info "| 200 | 131.876µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 -adapter_4 | 01/18 04:57:44.696261 00000039 TAOS_ADAPTER info "| 200 | 162.173µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=18 -adapter_2 | 01/18 04:57:44.707414 00000039 TAOS_ADAPTER info "| 200 | 164.419µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 -adapter_1 | 01/18 04:57:44.720842 00000039 TAOS_ADAPTER info "| 200 | 179.374µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 -adapter_3 | 01/18 04:57:44.732184 00000040 TAOS_ADAPTER info "| 200 | 141.174µs | 172.21.0.9 | POST | /rest/sql " sessionID=19 model=web -adapter_4 | 01/18 04:57:44.744024 00000039 TAOS_ADAPTER info "| 200 | 159.774µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=19 -adapter_2 | 01/18 04:57:44.773732 00000039 TAOS_ADAPTER info "| 200 | 178.993µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=21 -adapter_1 | 01/18 04:57:44.796518 00000039 TAOS_ADAPTER info "| 200 | 238.24µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 -adapter_3 | 01/18 04:57:44.810744 00000040 TAOS_ADAPTER info "| 200 | 176.133µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 -adapter_4 | 01/18 04:57:44.826395 00000039 TAOS_ADAPTER info "| 200 | 149.215µs | 172.21.0.9 | POST | /rest/sql " model=web sessionID=20 -``` - -`6044/udp` is the [StatsD](https://github.com/statsd/statsd)-compatible port, you can verify this feature with `nc` command(usually provided by `netcat` package). - -```bash -echo "foo:1|c" | nc -u -w0 127.0.0.1 6044 -``` - -Check the result in `taos` shell with `docker-compose exec`: +## Deploy with docker swarm -```bash -$ dc exec td-1 taos +If you want to deploy a container-based TDengine cluster on multiple hosts, you can use docker swarm. First, to establish a docker swarm cluster on these hosts, please refer to the official docker documentation. -Welcome to the TDengine shell from Linux, Client Version:2.4.0.0 -Copyright (c) 2020 by TAOS Data, Inc. All rights reserved. +The docker-compose file can refer to the previous section. Here is the command to start TDengine with docker swarm: -taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-01-18 04:37:42.902 | 17 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | - statsd | 2022-01-18 04:45:02.563 | 1 | 1 | 2 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | -Query OK, 2 row(s) in set (0.001838s) - -taos> select * from statsd.foo; - ts | value | metric_type | -======================================================================================= - 2022-01-18 04:45:02.563422822 | 1 | counter | -Query OK, 1 row(s) in set (0.003854s) -``` - -Use `docker-compose up -d adapter=1 to reduce the instances to 1 - -### Deploy TDengine cluster in Docker Swarm with `docker-compose.yml` - -If you use docker swarm mode, it will schedule arbitrator/taosd/taosadapter services into different hosts automatically. If you've no experience with k8s/kubernetes, this is the most convenient way to scale out the TDengine cluster with multiple hosts/servers. - -Use the `docker-compose.yml` file in previous use case, and deploy with `docker stack` or `docker deploy`: - -```bash -$ VERSION=2.4.0 docker stack deploy -c docker-compose.yml taos +```shell +$ VERSION=3.0.0.0 docker stack deploy -c docker-compose.yml taos Creating network taos_inter Creating network taos_api Creating service taos_arbitrator @@ -607,58 +468,40 @@ Creating service taos_adapter Creating service taos_nginx ``` -Now you've created a TDengine cluster with multiple host servers. - -Use `docker service` or `docker stack` to manage the cluster: +Checking status: - - -```bash +```shell $ docker stack ps taos ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS -79ni8temw59n taos_nginx.1 nginx:latest TM1701 Running Running about a minute ago -3e94u72msiyg taos_adapter.1 tdengine/tdengine:2.4.0 TM1702 Running Running 56 seconds ago -100amjkwzsc6 taos_td-2.1 tdengine/tdengine:2.4.0 TM1703 Running Running about a minute ago -pkjehr2vvaaa taos_td-1.1 tdengine/tdengine:2.4.0 TM1704 Running Running 2 minutes ago -tpzvgpsr1qkt taos_arbitrator.1 tdengine/tdengine:2.4.0 TM1705 Running Running 2 minutes ago -rvss3g5yg6fa taos_adapter.2 tdengine/tdengine:2.4.0 TM1706 Running Running 56 seconds ago -i2augxamfllf taos_adapter.3 tdengine/tdengine:2.4.0 TM1707 Running Running 56 seconds ago -lmjyhzccpvpg taos_adapter.4 tdengine/tdengine:2.4.0 TM1708 Running Running 56 seconds ago +79ni8temw59n taos_nginx.1 nginx:latest TM1701 Running Running about a minute ago +3e94u72msiyg taos_adapter.1 tdengine/tdengine:3.0.0.0 TM1702 Running Running 56 seconds ago +100amjkwzsc6 taos_td-2.1 tdengine/tdengine:3.0.0.0 TM1703 Running Running about a minute ago +pkjehr2vvaaa taos_td-1.1 tdengine/tdengine:3.0.0.0 TM1704 Running Running 2 minutes ago +tpzvgpsr1qkt taos_arbitrator.1 tdengine/tdengine:3.0.0.0 TM1705 Running Running 2 minutes ago +rvss3g5yg6fa taos_adapter.2 tdengine/tdengine:3.0.0.0 TM1706 Running Running 56 seconds ago +i2augxamfllf taos_adapter.3 tdengine/tdengine:3.0.0.0 TM1707 Running Running 56 seconds ago +lmjyhzccpvpg taos_adapter.4 tdengine/tdengine:3.0.0.0 TM1708 Running Running 56 seconds ago $ docker service ls ID NAME MODE REPLICAS IMAGE PORTS -561t4lu6nfw6 taos_adapter replicated 4/4 tdengine/tdengine:2.4.0 -3hk5ct3q90sm taos_arbitrator replicated 1/1 tdengine/tdengine:2.4.0 +561t4lu6nfw6 taos_adapter replicated 4/4 tdengine/tdengine:3.0.0.0 +3hk5ct3q90sm taos_arbitrator replicated 1/1 tdengine/tdengine:3.0.0.0 d8qr52envqzu taos_nginx replicated 1/1 nginx:latest *:6041->6041/tcp, *:6044->6044/udp -2isssfvjk747 taos_td-1 replicated 1/1 tdengine/tdengine:2.4.0 -9pzw7u02ichv taos_td-2 replicated 1/1 tdengine/tdengine:2.4.0 +2isssfvjk747 taos_td-1 replicated 1/1 tdengine/tdengine:3.0.0.0 +9pzw7u02ichv taos_td-2 replicated 1/1 tdengine/tdengine:3.0.0.0 ``` - - -It shows that there are two dnodes, one arbitrator, four taosadapter and one nginx reverse-forward service in this cluster. +From the above output, you can see two dnodes, two taosAdapters, and one Nginx reverse proxy service. -You can scale down the taosadapter replicas to `1` by `docker service`: +Next, we can reduce the number of taosAdapter services. -```bash +```shell $ docker service scale taos_adapter=1 taos_adapter scaled to 1 -overall progress: 1 out of 1 tasks -1/1: running [==================================================>] +overall progress: 1 out of 1 tasks +1/1: running [==================================================>] verify: Service converged $ docker service ls -f name=taos_adapter ID NAME MODE REPLICAS IMAGE PORTS -561t4lu6nfw6 taos_adapter replicated 1/1 tdengine/tdengine:2.4.0 -``` - -Now it remains only 1 taosadapter instance in the cluster. - -When you want to remove the cluster, just type: - -```bash -docker stack rm taos +561t4lu6nfw6 taos_adapter replicated 1/1 tdengine/tdengine:3.0.0.0 ``` - -### Environment Variables - -When you start `tdengine` image, you can adjust the configuration of TDengine by passing environment variables on the `docker run` command line or in the docker compose file. You can use all of the environment variables that passed to taosd or taosadapter. diff --git a/packaging/docker/docker-compose.yml b/packaging/docker/docker-compose.yml deleted file mode 100644 index 301b41e7d43c2a894d866c1f0d45cf8d13328585..0000000000000000000000000000000000000000 --- a/packaging/docker/docker-compose.yml +++ /dev/null @@ -1,77 +0,0 @@ -version: "3" - -networks: - inter: - api: - -services: - arbitrator: - image: tdengine/tdengine:$VERSION - command: tarbitrator - networks: - - inter - td-1: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-1" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td1:/var/lib/taos/ - - taoslog-td1:/var/log/taos/ - td-2: - image: tdengine/tdengine:$VERSION - networks: - - inter - environment: - TAOS_FQDN: "td-2" - TAOS_FIRST_EP: "td-1" - TAOS_NUM_OF_MNODES: "2" - TAOS_REPLICA: "2" - TAOS_ARBITRATOR: arbitrator:6042 - volumes: - - taosdata-td2:/var/lib/taos/ - - taoslog-td2:/var/log/taos/ - adapter: - image: tdengine/tdengine:$VERSION - command: taosadapter - networks: - - inter - environment: - TAOS_FIRST_EP: "td-1" - TOAS_SECOND_EP: "td-2" - deploy: - replicas: 4 - update_config: - parallelism: 4 - nginx: - image: nginx - depends_on: - - adapter - networks: - - inter - - api - ports: - - 6041:6041 - - 6044:6044/udp - command: [ - "sh", - "-c", - "while true; - do curl -s http://adapter:6041/-/ping >/dev/null && break; - done; - printf 'server{listen 6041;location /{proxy_pass http://adapter:6041;}}' - > /etc/nginx/conf.d/rest.conf; - printf 'stream{server{listen 6044 udp;proxy_pass adapter:6044;}}' - >> /etc/nginx/nginx.conf;cat /etc/nginx/nginx.conf; - nginx -g 'daemon off;'", - ] -volumes: - taosdata-td1: - taoslog-td1: - taosdata-td2: - taoslog-td2: diff --git a/packaging/docker/dockerManifest.sh b/packaging/docker/dockerManifest.sh index 8f71e30fbdca1cc9adf8e9b46c652475822e4b08..db71bf8833d3fce82a461e7021da3eaa66b0bf60 100755 --- a/packaging/docker/dockerManifest.sh +++ b/packaging/docker/dockerManifest.sh @@ -1,9 +1,9 @@ #!/bin/bash set -e #set -x -set -v +set -v -# dockerbuild.sh +# dockerbuild.sh # -n [version number] # -p [xxxx] # -V [stable | beta] @@ -28,7 +28,7 @@ do V) #echo "verType=$OPTARG" verType=$(echo $OPTARG) - ;; + ;; h) echo "Usage: `basename $0` -n [version number] " echo " -p [password for docker hub] " @@ -39,8 +39,8 @@ do a) #echo "dockerLatest=$OPTARG" dockerLatest=$(echo $OPTARG) - ;; - ?) #unknow option + ;; + ?) #unknow option echo "unkonw argument" exit 1 ;; @@ -60,7 +60,7 @@ if [ "$verType" == "stable" ]; then elif [ "$verType" == "beta" ];then verType=beta tagVal=ver-${version}-beta - dockerinput=TDengine-server-${version}-${verType}-Linux-$cpuType.tar.gz + dockerinput=TDengine-server-${version}-${verType}-Linux-$cpuType.tar.gz dockerinput_x64=TDengine-server-${version}-${verType}-Linux-amd64.tar.gz dockerim=tdengine/tdengine-beta dockeramd64=tdengine/tdengine-amd64-beta @@ -73,30 +73,30 @@ fi username="tdengine" -# generate docker verison +# generate docker version echo "generate ${dockerim}:${version}" docker manifest create -a ${dockerim}:${version} ${dockeramd64}:${version} ${dockeraarch64}:${version} docker manifest inspect ${dockerim}:${version} docker manifest rm ${dockerim}:${version} docker manifest create -a ${dockerim}:${version} ${dockeramd64}:${version} ${dockeraarch64}:${version} docker manifest inspect ${dockerim}:${version} -docker login -u ${username} -p ${passWord} +docker login -u ${username} -p ${passWord} docker manifest push ${dockerim}:${version} -# generate docker latest +# generate docker latest echo "generate ${dockerim}:latest " if [ ${dockerLatest} == 'y' ] ;then echo "docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest" docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest - docker manifest inspect ${dockerim}:latest - docker manifest rm ${dockerim}:latest + docker manifest inspect ${dockerim}:latest + docker manifest rm ${dockerim}:latest docker manifest create -a ${dockerim}:latest ${dockeramd64}:latest ${dockeraarch64}:latest docker manifest inspect ${dockerim}:latest docker login -u tdengine -p ${passWord} #replace the docker registry username and password docker manifest push ${dockerim}:latest - docker pull tdengine/tdengine:latest + docker pull tdengine/tdengine:latest fi diff --git a/packaging/docker/dockerbuild.sh b/packaging/docker/dockerbuild.sh index b02387a3d1191c09dadfa1ce21ab33183a52a7da..4b6fc8576b233192a6835d455d73baf14abc1758 100755 --- a/packaging/docker/dockerbuild.sh +++ b/packaging/docker/dockerbuild.sh @@ -74,7 +74,7 @@ do done -# Check_verison() +# Check_version() # { # } @@ -102,14 +102,14 @@ scriptDir=$(dirname $(readlink -f $0)) communityDir=${scriptDir}/../../../community DockerfilePath=${communityDir}/packaging/docker/ if [ "$cloudBuild" == "y" ]; then - comunityArchiveDir=/nas/TDengine/v$version/cloud + communityArchiveDir=/nas/TDengine/v$version/cloud Dockerfile=${communityDir}/packaging/docker/DockerfileCloud else - comunityArchiveDir=/nas/TDengine/v$version/community + communityArchiveDir=/nas/TDengine/v$version/community Dockerfile=${communityDir}/packaging/docker/Dockerfile fi cd ${scriptDir} -cp -f ${comunityArchiveDir}/${pkgFile} . +cp -f ${communityArchiveDir}/${pkgFile} . echo "dirName=${dirName}" diff --git a/packaging/release.sh b/packaging/release.sh deleted file mode 100755 index 1dfbf2b1124b0ebdeb1d9062c5d9d659eabe8a78..0000000000000000000000000000000000000000 --- a/packaging/release.sh +++ /dev/null @@ -1,319 +0,0 @@ -#!/bin/bash -# -# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os - -set -e -# set -x - -# release.sh -v [cluster | edge] -# -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64...] -# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -# -V [stable | beta] -# -l [full | lite] -# -s [static | dynamic] -# -d [taos | ...] -# -n [2.0.0.3] -# -m [2.0.0.0] -# -H [ false | true] - -# set parameters by default value -verMode=edge # [cluster, edge, cloud] -verType=stable # [stable, beta] -cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 loongarch64...] -osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] -pagMode=full # [full | lite] -soMode=dynamic # [static | dynamic] -dbName=taos # [taos | ...] -allocator=glibc # [glibc | jemalloc] -verNumber="" -verNumberComp="3.0.0.0" -httpdBuild=false - -while getopts "hv:V:c:o:l:s:d:a:n:m:H:" arg; do - case $arg in - v) - #echo "verMode=$OPTARG" - verMode=$(echo $OPTARG) - ;; - V) - #echo "verType=$OPTARG" - verType=$(echo $OPTARG) - ;; - c) - #echo "cpuType=$OPTARG" - cpuType=$(echo $OPTARG) - ;; - l) - #echo "pagMode=$OPTARG" - pagMode=$(echo $OPTARG) - ;; - s) - #echo "soMode=$OPTARG" - soMode=$(echo $OPTARG) - ;; - d) - #echo "dbName=$OPTARG" - dbName=$(echo $OPTARG) - ;; - a) - #echo "allocator=$OPTARG" - allocator=$(echo $OPTARG) - ;; - n) - #echo "verNumber=$OPTARG" - verNumber=$(echo $OPTARG) - ;; - m) - #echo "verNumberComp=$OPTARG" - verNumberComp=$(echo $OPTARG) - ;; - o) - #echo "osType=$OPTARG" - osType=$(echo $OPTARG) - ;; - H) - #echo "httpdBuild=$OPTARG" - httpdBuild=$(echo $OPTARG) - ;; - h) - echo "Usage: $(basename $0) -v [cluster | edge] " - echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 | loongarch64 ...] " - echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " - echo " -V [stable | beta] " - echo " -l [full | lite] " - echo " -a [glibc | jemalloc] " - echo " -s [static | dynamic] " - echo " -d [taos | ...] " - echo " -n [version number] " - echo " -m [compatible version number] " - echo " -H [false | true] " - exit 0 - ;; - ?) #unknow option - echo "unkonw argument" - exit 1 - ;; - esac -done - -osType=$(uname) - -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} dbName=${dbName} allocator=${allocator} verNumber=${verNumber} verNumberComp=${verNumberComp} httpdBuild=${httpdBuild}" - -curr_dir=$(pwd) - -if [ "$osType" == "Darwin" ]; then - script_dir=$(dirname $0) - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. -else - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" -fi - -csudo="" -#if command -v sudo > /dev/null; then -# csudo="sudo " -#fi - -function is_valid_version() { - [ -z $1 ] && return 1 || : - - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 -} - -function vercomp() { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i = ${#ver1[@]}; i < ${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i = 0; i < ${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done - echo 0 -} - -# 1. check version information -if ( (! is_valid_version $verNumber) || (! is_valid_version $verNumberComp) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then - echo "please enter correct version" - exit 0 -fi - -echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" - -build_time=$(date +"%F %R") - -# get commint id from git -gitinfo=$(git rev-parse --verify HEAD) - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - gitinfoOfInternal=$(git rev-parse --verify HEAD) -else - gitinfoOfInternal=NULL -fi - -cd "${curr_dir}" - -# 2. cmake executable file -compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - rm -rf ${compile_dir} -fi - -mkdir -p ${compile_dir} -cd ${compile_dir} - -if [[ "$allocator" == "jemalloc" ]]; then - allocator_macro="-DJEMALLOC_ENABLED=true" -else - allocator_macro="" -fi - -if [[ "$dbName" != "taos" ]]; then - source ${enterprise_dir}/packaging/oem/sed_$dbName.sh - replace_community_$dbName -fi - -if [[ "$httpdBuild" == "true" ]]; then - BUILD_HTTP=true -else - BUILD_HTTP=false -fi - -if [[ "$verMode" == "cluster" ]] || [[ "$verMode" == "cloud" ]]; then - BUILD_HTTP=internal -fi - -if [[ "$pagMode" == "full" ]]; then - BUILD_TOOLS=true -else - BUILD_TOOLS=false -fi - -# check support cpu type -if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "arm64" ]] || [[ "$cpuType" == "arm32" ]] || [[ "$cpuType" == "mips64" ]] || [[ "$cpuType" == "loongarch64" ]] ; then - if [ "$verMode" == "edge" ]; then - # community-version compile - cmake ../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cloud" ]; then - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DBUILD_CLOUD=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - elif [ "$verMode" == "cluster" ]; then - if [[ "$dbName" != "taos" ]]; then - replace_enterprise_$dbName - fi - cmake ../../ -DCPUTYPE=${cpuType} -DWEBSOCKET=true -DBUILD_TAOSX=true -DOSTYPE=${osType} -DSOMODE=${soMode} -DDBNAME=${dbName} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DBUILD_HTTP=${BUILD_HTTP} -DBUILD_TOOLS=${BUILD_TOOLS} ${allocator_macro} - fi -else - echo "input cpuType=${cpuType} error!!!" - exit 1 -fi - -ostype=`uname` -if [ "${ostype}" == "Darwin" ]; then - CORES=$(sysctl -n hw.ncpu) -else - CORES=$(grep -c ^processor /proc/cpuinfo) -fi - -if [[ "$allocator" == "jemalloc" ]]; then - # jemalloc need compile first, so disable parallel build - make -j ${CORES} && ${csudo}make install -else - make -j ${CORES} && ${csudo}make install -fi - -cd ${curr_dir} - -# 3. Call the corresponding script for packaging -if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$verMode" != "cloud" ]] && [[ "$pagMode" == "full" ]] && [[ "$cpuType" == "x64" ]] && [[ "$dbName" == "taos" ]]; then - ret='0' - command -v dpkg >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then - cd ${top_dir}/tools/taos-tools/packaging/deb - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-deb.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp} - fi - fi - else - echo "==========dpkg command not exist, so not release deb package!!!" - fi - ret='0' - command -v rpmbuild >/dev/null 2>&1 || { ret='1'; } - if [ "$ret" -eq 0 ]; then - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - rm -rf ${output_dir} - fi - mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} - - if [[ "$pagMode" == "full" ]]; then - if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then - cd ${top_dir}/tools/taos-tools/packaging/rpm - taos_tools_ver=$(git tag |grep -v taos | sort | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - - ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ - ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} ${verNumberComp} - fi - fi - else - echo "==========rpmbuild command not exist, so not release rpm package!!!" - fi - fi - - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} - -else - cd ${script_dir}/tools - ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} - ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} -fi diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index dfdbaa6fdd634679d45b1fa3eeaa5aabdea36a23..5aeff0e2fac3cde9e381a33719f5ab25722e16f5 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -35,6 +35,20 @@ dumpName="taosdump" demoName="taosdemo" xname="taosx" +clientName2="taos" +serverName2="taosd" +productName2="TDengine" +emailName2="taosdata.com" + +benchmarkName2="${clientName2}Benchmark" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" + +historyFile="${clientName2}_history" +logDir="/var/log/${clientName2}" +configDir="/etc/${clientName2}" +installDir="/usr/local/${clientName}" + data_dir=${dataDir} log_dir=${logDir} cfg_install_dir=${configDir} @@ -217,6 +231,13 @@ function install_bin() { [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + + if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : + [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript2} || : + fi } function install_lib() { @@ -518,7 +539,7 @@ function install_config() { local_fqdn_check echo - echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName}:6030) of an existing ${productName} cluster node to join${NC}" + echo -e -n "${GREEN}Enter FQDN:port (like h1.${emailName2}:6030) of an existing ${productName2} cluster node to join${NC}" echo echo -e -n "${GREEN}OR leave it blank to build one${NC}:" read firstEp @@ -655,6 +676,9 @@ function clean_service_on_systemd() { fi ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null ${csudo}rm -f ${tarbitratord_service_config} + # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + # ${csudo}rm -f ${service_config_dir}/${serverName2}.service + # fi } function install_service_on_systemd() { @@ -663,6 +687,13 @@ function install_service_on_systemd() { [ -f ${script_dir}/cfg/${serverName}.service ] && ${csudo}cp ${script_dir}/cfg/${serverName}.service \ ${service_config_dir}/ || : + + # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + # [ -f ${script_dir}/cfg/${serverName}.service ] && + # ${csudo}cp ${script_dir}/cfg/${serverName}.service \ + # ${service_config_dir}/${serverName2}.service || : + # fi + ${csudo}systemctl daemon-reload ${csudo}systemctl enable ${serverName} @@ -793,7 +824,7 @@ function updateProduct() { tar -zxf ${tarName} install_jemalloc - echo -e "${GREEN}Start to update ${productName}...${NC}" + echo -e "${GREEN}Start to update ${productName2}...${NC}" # Stop the service if running if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then if ((${service_mod} == 0)); then @@ -830,25 +861,25 @@ function updateProduct() { echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" + echo -e "${GREEN_DARK}To configure Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" + echo -e "${GREEN_DARK}To start Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" + echo -e "${GREEN_DARK}To start Adapter ${NC}: ${csudo}service taosadapter start${NC}" else echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" [ -f ${installDir}/bin/taosadapter ] && \ - echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" + echo -e "${GREEN_DARK}To start ${clientName} Adapter ${NC}: taosadapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" else - echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName} -h $serverFqdn${NC} in shell${NC}" + echo -e "${GREEN_DARK}To access ${productName2} ${NC}: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}" fi if ((${prompt_force} == 1)); then @@ -856,13 +887,13 @@ function updateProduct() { echo -e "${RED}Please run '${serverName} --force-keep-file' at first time for the exist ${productName} $exist_version!${NC}" fi echo - echo -e "\033[44;32;1m${productName} is updated successfully!${NC}" + echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" else install_bin install_config echo - echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi rm -rf $(tar -tf ${tarName} | grep -v "^\./$") @@ -876,7 +907,7 @@ function installProduct() { fi tar -zxf ${tarName} - echo -e "${GREEN}Start to install ${productName}...${NC}" + echo -e "${GREEN}Start to install ${productName2}...${NC}" install_main_path @@ -965,7 +996,7 @@ serverFqdn=$(hostname) if [ "$verType" == "server" ]; then # Check default 2.x data file. if [ -x ${data_dir}/dnode/dnodeCfg.json ]; then - echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of tdengine 2.x, please clear it before installing!\033[0m" + echo -e "\033[44;31;5mThe default data directory ${data_dir} contains old data of ${productName2} 2.x, please clear it before installing!\033[0m" else # Install server and client if [ -x ${bin_dir}/${serverName} ]; then diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index f7d677c9821ed685750dcd50e2823f9abda4ef42..1543c59297d663f0579d53f5c9778385cbef3ba8 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -23,6 +23,16 @@ osType=Linux pagMode=full verMode=edge +clientName2="taos" +serverName2="taosd" +productName2="TDengine" +emailName2="taosdata.com" + +benchmarkName2="${clientName2}Benchmark" +dumpName2="${clientName2}dump" +demoName2="${clientName2}demo" +uninstallScript2="rm${clientName2}" + if [ "$osType" != "Darwin" ]; then script_dir=$(dirname $(readlink -f "$0")) # Dynamic directory @@ -112,6 +122,15 @@ function install_bin() { fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : + + if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + #Make link + [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName2} || : + if [ "$osType" != "Darwin" ]; then + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/${demoName2} || : + fi + [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : + fi } function clean_lib() { @@ -263,9 +282,9 @@ function update_TDengine() { exit 1 fi tar -zxf ${tarName} - echo -e "${GREEN}Start to update ${productName} client...${NC}" + echo -e "${GREEN}Start to update ${productName2} client...${NC}" # Stop the client shell if running - if ps aux | grep -v grep | grep ${clientName} &> /dev/null; then + if ps aux | grep -v grep | grep ${clientName2} &> /dev/null; then kill_client sleep 1 fi @@ -284,7 +303,7 @@ function update_TDengine() { install_config echo - echo -e "\033[44;32;1m${productName} client is updated successfully!${NC}" + echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" rm -rf $(tar -tf ${tarName}) } @@ -296,7 +315,7 @@ function install_TDengine() { exit 1 fi tar -zxf ${tarName} - echo -e "${GREEN}Start to install ${productName} client...${NC}" + echo -e "${GREEN}Start to install ${productName2} client...${NC}" install_main_path install_log @@ -311,7 +330,7 @@ function install_TDengine() { install_config echo - echo -e "\033[44;32;1m${productName} client is installed successfully!${NC}" + echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}" rm -rf $(tar -tf ${tarName}) } @@ -321,7 +340,7 @@ function install_TDengine() { # Install or updata client and client # if server is already install, don't install client if [ -e ${bin_dir}/${serverName} ]; then - echo -e "\033[44;32;1mThere are already installed ${productName} server, so don't need install client!${NC}" + echo -e "\033[44;32;1mThere are already installed ${productName2} server, so don't need install client!${NC}" exit 0 fi diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index 3b6d610e882e9d2b6d9d8986f23fd1a5b8f9cb0a..a428c612b2607ea482f0a6ddc5b26636470b72b6 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,6 +1,9 @@ -TDengine is a high-efficient, scalable, high-available distributed time-series database, which makes a lot of optimizations on inserting and querying data, which is far more efficient than normal regular databases. So TDengine can meet the high requirements of IOT and other areas on storing and querying a large amount of data. +TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. To configure TDengine : edit /etc/taos/taos.cfg To start service : launchctl start com.tdengine.taosd To start Taos Adapter : launchctl start com.tdengine.taosadapter -To access TDengine : use taos in shell \ No newline at end of file +To access TDengine : use taos in shell + +If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. + diff --git a/packaging/tools/mac_before_install_client.txt b/packaging/tools/mac_before_install_client.txt new file mode 100644 index 0000000000000000000000000000000000000000..0457d73c49a0bdba49864477f32d07aaa512c375 --- /dev/null +++ b/packaging/tools/mac_before_install_client.txt @@ -0,0 +1,9 @@ +TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. + +Once it's installed, please take the steps below: +1: open a terminal/shell in Mac +2: if connecting to Cloud Service, follow the instructions on your cloud service account and configure the environment variable +3: if connecting to another TDengine Service, you can also view help information via "taos --help" +4: execute command taos + +If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. diff --git a/packaging/tools/make_install.bat b/packaging/tools/make_install.bat index 993a8a19ee2862addc68304ff36359062846145b..f5ed1cdf666c2442f10b9d45320f4a93a6cdfa6f 100644 --- a/packaging/tools/make_install.bat +++ b/packaging/tools/make_install.bat @@ -22,6 +22,7 @@ set binary_dir=%3 set binary_dir=%binary_dir:/=\\% set osType=%4 set verNumber=%5 +set Enterprise=%6 set target_dir=C:\\TDengine if not exist %target_dir% ( @@ -65,7 +66,33 @@ if exist %binary_dir%\\build\\lib\\taosws.dll ( if exist %binary_dir%\\build\\bin\\taosdump.exe ( copy %binary_dir%\\build\\bin\\taosdump.exe %target_dir% > nul ) - +if %Enterprise% == TRUE ( + if exist %binary_dir%\\build\\bin\\taosx.exe ( + copy %binary_dir%\\build\\bin\\taosx.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_sim.exe ( + copy %binary_dir%\\build\\bin\\tmq_sim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tsim.exe ( + copy %binary_dir%\\build\\bin\\tsim.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_taosx_ci.exe ( + copy %binary_dir%\\build\\bin\\tmq_taosx_ci.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\tmq_demo.exe ( + copy %binary_dir%\\build\\bin\\tmq_demo.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\dumper.exe ( + copy %binary_dir%\\build\\bin\\dumper.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\runUdf.exe ( + copy %binary_dir%\\build\\bin\\runUdf.exe %target_dir% > nul + ) + if exist %binary_dir%\\build\\bin\\create_table.exe ( + copy %binary_dir%\\build\\bin\\create_table.exe %target_dir% > nul + ) +) + copy %binary_dir%\\build\\bin\\taosd.exe %target_dir% > nul copy %binary_dir%\\build\\bin\\udfd.exe %target_dir% > nul if exist %binary_dir%\\build\\bin\\taosadapter.exe ( diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index aae3c3b593ba9281929f793145b4598b15f46c95..5725560bd61266bf05c2e162652b271382e412c0 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # This file is used to install TAOS time-series database on linux systems. The operating system # is required to use systemd to manage services at boot @@ -340,7 +340,7 @@ function install_lib() { #install_avro lib64 if [ "$osType" != "Darwin" ]; then - ${csudo}ldconfig + ${csudo}ldconfig /etc/ld.so.conf.d fi } diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index db28de8decada6593abf5600bd8cf86cfe7d0984..208bfc183c91bba96be689fe6a475a5ae6e38649 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -13,7 +13,13 @@ osType=$5 verMode=$6 verType=$7 pagMode=$8 -dbName=$9 +#comVersion=$9 +dbName=$10 + +productName2="${11}" +#serverName2="${12}d" +clientName2="${12}" +# cusEmail2=${13} productName="TDengine" clientName="taos" @@ -38,15 +44,21 @@ release_dir="${top_dir}/release" #package_name='linux' if [ "$verMode" == "cluster" ]; then - install_dir="${release_dir}/${productName}-enterprise-client-${version}" + install_dir="${release_dir}/${productName2}-enterprise-client-${version}" elif [ "$verMode" == "cloud" ]; then - install_dir="${release_dir}/${productName}-cloud-client-${version}" + install_dir="${release_dir}/${productName2}-cloud-client-${version}" else - install_dir="${release_dir}/${productName}-client-${version}" + install_dir="${release_dir}/${productName2}-client-${version}" fi # Directories and files. +if [ "$verMode" == "cluster" ]; then + sed -i 's/verMode=edge/verMode=cluster/g' ${script_dir}/remove_client.sh + sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" ${script_dir}/remove_client.sh + sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" ${script_dir}/remove_client.sh +fi + if [ "$osType" != "Darwin" ]; then if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/${clientName} @@ -131,22 +143,28 @@ fi cd ${curr_dir} cp ${install_files} ${install_dir} +cp ${install_dir}/install_client.sh install_client_temp.sh if [ "$osType" == "Darwin" ]; then - sed 's/osType=Linux/osType=Darwin/g' ${install_dir}/install_client.sh >>install_client_temp.sh + sed -i 's/osType=Linux/osType=Darwin/g' install_client_temp.sh mv install_client_temp.sh ${install_dir}/install_client.sh fi if [ "$verMode" == "cluster" ]; then - sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install_client.sh >>install_client_temp.sh + sed -i 's/verMode=edge/verMode=cluster/g' install_client_temp.sh + sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_client_temp.sh + sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_client_temp.sh + sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_client_temp.sh + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_client_temp.sh + mv install_client_temp.sh ${install_dir}/install_client.sh fi if [ "$verMode" == "cloud" ]; then - sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install_client.sh >>install_client_temp.sh + sed -i 's/verMode=edge/verMode=cloud/g' install_client_temp.sh mv install_client_temp.sh ${install_dir}/install_client.sh fi if [ "$pagMode" == "lite" ]; then - sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install_client.sh >>install_client_temp.sh + sed -i 's/pagMode=full/pagMode=lite/g' install_client_temp.sh mv install_client_temp.sh ${install_dir}/install_client.sh fi chmod a+x ${install_dir}/install_client.sh diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 0ee548242f2a69b025dddb46381d8d8c1d30d799..0a34d81b7fa0000ad8924c3968ae897a6f7f43f3 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -16,6 +16,10 @@ verType=$7 pagMode=$8 versionComp=$9 dbName=${10} +productName2="${11}" +serverName2="${12}d" +clientName2="${12}" +cusEmail2="${13}" script_dir="$(dirname $(readlink -f $0))" top_dir="$(readlink -f ${script_dir}/../..)" @@ -38,11 +42,11 @@ release_dir="${top_dir}/release" #package_name='linux' if [ "$verMode" == "cluster" ]; then - install_dir="${release_dir}/${productName}-enterprise-server-${version}" + install_dir="${release_dir}/${productName2}-enterprise-server-${version}" elif [ "$verMode" == "cloud" ]; then - install_dir="${release_dir}/${productName}-cloud-server-${version}" + install_dir="${release_dir}/${productName2}-cloud-server-${version}" else - install_dir="${release_dir}/${productName}-server-${version}" + install_dir="${release_dir}/${productName2}-server-${version}" fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then @@ -50,11 +54,11 @@ if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" taostools_ver=$(git tag |grep -v taos | sort | tail -1) - taostools_install_dir="${release_dir}/${clientName}Tools-${taostools_ver}" + taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} else - taostools_install_dir="${release_dir}/${clientName}Tools-${version}" + taostools_install_dir="${release_dir}/${clientName2}Tools-${version}" fi # Directories and files @@ -225,6 +229,10 @@ fi if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >>remove_temp.sh + sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" remove_temp.sh + sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" remove_temp.sh + sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" remove_temp.sh + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" remove_temp.sh mv remove_temp.sh ${install_dir}/bin/remove.sh fi if [ "$verMode" == "cloud" ]; then @@ -247,16 +255,21 @@ fi cd ${curr_dir} cp ${install_files} ${install_dir} +cp ${install_dir}/install.sh install_temp.sh if [ "$verMode" == "cluster" ]; then - sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/install.sh >>install_temp.sh + sed -i 's/verMode=edge/verMode=cluster/g' install_temp.sh + sed -i "s/serverName2=\"taosd\"/serverName2=\"${serverName2}\"/g" install_temp.sh + sed -i "s/clientName2=\"taos\"/clientName2=\"${clientName2}\"/g" install_temp.sh + sed -i "s/productName2=\"TDengine\"/productName2=\"${productName2}\"/g" install_temp.sh + sed -i "s/emailName2=\"taosdata.com\"/emailName2=\"${cusEmail2}\"/g" install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$verMode" == "cloud" ]; then - sed 's/verMode=edge/verMode=cloud/g' ${install_dir}/install.sh >>install_temp.sh + sed -i 's/verMode=edge/verMode=cloud/g' install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi if [ "$pagMode" == "lite" ]; then - sed 's/pagMode=full/pagMode=lite/g' ${install_dir}/install.sh >>install_temp.sh + sed -i 's/pagMode=full/pagMode=lite/g' install_temp.sh mv install_temp.sh ${install_dir}/install.sh fi chmod a+x ${install_dir}/install.sh diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 4441e0ba1f54e61ab4a276eed33a7001c0845a22..78eb7f7587a3048071a2d5b14e40bc45429115d1 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -48,6 +48,7 @@ fi data_link_dir="${install_main_dir}/data" log_link_dir="${install_main_dir}/log" +install_log_path="${log_dir}/tdengine_install.log" # static directory cfg_dir="${install_main_dir}/cfg" @@ -101,6 +102,11 @@ if [ "$osType" = "Darwin" ]; then ${csudo}cp -rf tdengine ${install_main_dir} fi +function log_print(){ + now=$(date +"%D %T") + echo "$now $1" >> ${install_log_path} +} + function kill_taosadapter() { # ${csudo}pkill -f taosadapter || : pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') @@ -109,6 +115,13 @@ function kill_taosadapter() { fi } +function kill_taoskeeper() { + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function kill_taosd() { # ${csudo}pkill -f taosd || : pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') @@ -118,6 +131,7 @@ function kill_taosd() { } function install_include() { + log_print "start install include from ${inc_dir} to ${inc_link_dir}" ${csudo}mkdir -p ${inc_link_dir} ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : [ -f ${inc_link_dir}/taosws.h ] && ${csudo}rm -f ${inc_link_dir}/taosws.h ||: @@ -128,39 +142,45 @@ function install_include() { ${csudo}ln -s ${inc_dir}/taosudf.h ${inc_link_dir}/taosudf.h [ -f ${inc_dir}/taosws.h ] && ${csudo}ln -sf ${inc_dir}/taosws.h ${inc_link_dir}/taosws.h ||: + log_print "install include success" } function install_lib() { + log_print "start install lib from ${lib_dir} to ${lib_link_dir}" ${csudo}rm -f ${lib_link_dir}/libtaos* || : ${csudo}rm -f ${lib64_link_dir}/libtaos* || : [ -f ${lib_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib_link_dir}/libtaosws.${lib_file_ext} || : [ -f ${lib64_link_dir}/libtaosws.${lib_file_ext} ] && ${csudo}rm -f ${lib64_link_dir}/libtaosws.${lib_file_ext} || : - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} - ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib_link_dir}/libtaos.${lib_file_ext_1} ${lib_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib_link_dir}/libtaosws.${lib_file_ext} ||: if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.${lib_file_ext} ]]; then - ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} || : - ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} || : + ${csudo}ln -s ${lib_dir}/libtaos.* ${lib64_link_dir}/libtaos.${lib_file_ext_1} 2>>${install_log_path} || return 1 + ${csudo}ln -s ${lib64_link_dir}/libtaos.${lib_file_ext_1} ${lib64_link_dir}/libtaos.${lib_file_ext} 2>>${install_log_path} || return 1 - [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} || : + [ -f ${lib_dir}/libtaosws.${lib_file_ext} ] && ${csudo}ln -sf ${lib_dir}/libtaosws.${lib_file_ext} ${lib64_link_dir}/libtaosws.${lib_file_ext} 2>>${install_log_path} fi if [ "$osType" != "Darwin" ]; then ${csudo}ldconfig fi + + log_print "install lib success" } function install_bin() { # Remove links + log_print "start install bin from ${bin_dir} to ${bin_link_dir}" ${csudo}rm -f ${bin_link_dir}/taos || : ${csudo}rm -f ${bin_link_dir}/taosd || : ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/rmtaos || : @@ -169,16 +189,38 @@ function install_bin() { ${csudo}chmod 0555 ${bin_dir}/* #Make link - [ -x ${bin_dir}/taos ] && ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : - [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : - [ -x ${bin_dir}/udfd ] && ${csudo}ln -s ${bin_dir}/udfd ${bin_link_dir}/udfd || : - [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter || : - [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || : - [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark || : - [ -x ${bin_dir}/TDinsight.sh ] && ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : - [ -x ${bin_dir}/taosdump ] && ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : - [ -x ${bin_dir}/set_core.sh ] && ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${bin_dir}/remove.sh ] && ${csudo}ln -s ${bin_dir}/remove.sh ${bin_link_dir}/rmtaos || : + if [ -x ${bin_dir}/taos ]; then + ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/taosd ]; then + ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/udfd ]; then + ${csudo}ln -s ${bin_dir}/udfd ${bin_link_dir}/udfd 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/taosadapter ]; then + ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/taosBenchmark ]; then + ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo 2>>${install_log_path} + ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark 2>>${install_log_path} + fi + if [ -x ${bin_dir}/TDinsight.sh ]; then + ${csudo}ln -sf ${bin_dir}/TDinsight.sh ${bin_link_dir}/TDinsight.sh 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/taosdump ]; then + ${csudo}ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/set_core.sh ]; then + ${csudo}ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/remove.sh ]; then + ${csudo}ln -s ${bin_dir}/remove.sh ${bin_link_dir}/rmtaos 2>>${install_log_path} || return 1 + fi + if [ -x ${bin_dir}/taoskeeper ]; then + ${csudo}ln -sf ${bin_dir}/taoskeeper ${bin_link_dir}/taoskeeper 2>>${install_log_path} || return 1 + fi + log_print "install bin success" } function add_newHostname_to_hosts() { @@ -351,7 +393,24 @@ function install_taosadapter_config() { ${csudo}ln -s ${cfg_install_dir}/taosadapter.toml ${cfg_dir} } +function install_taoskeeper_config() { + if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then + [ ! -d %{cfg_install_dir} ] && + ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml + fi + + [ -f ${cfg_dir}/keeper.toml ] && + ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new + + [ -f ${cfg_install_dir}/keeper.toml ] && + ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir} +} + function install_config() { + log_print "start install config from ${cfg_dir} to ${cfg_install_dir}" if [ ! -f "${cfg_install_dir}/taos.cfg" ]; then ${csudo}${csudo}mkdir -p ${cfg_install_dir} [ -f ${cfg_dir}/taos.cfg ] && ${csudo}cp ${cfg_dir}/taos.cfg ${cfg_install_dir} @@ -419,6 +478,8 @@ function install_config() { break fi done + + log_print "install config success" } function clean_service_on_sysvinit() { @@ -533,6 +594,7 @@ function install_taosadapter_service() { } function install_service() { + log_print "start install service" if [ "$osType" != "Darwin" ]; then if ((${service_mod}==0)); then install_service_on_systemd @@ -546,6 +608,7 @@ function install_service() { else install_service_on_launchctl fi + log_print "install service success" } function install_app() { @@ -564,8 +627,16 @@ function install_app() { fi } -function install_TDengine() { - echo -e "${GREEN}Start to install TDengine...${NC}" +function checkDirectory() { + if [ ! -d "${bin_link_dir}" ]; then + ${csudo}mkdir -p ${bin_link_dir} + log_print "${bin_link_dir} directory created" + fi + + if [ ! -d "${lib_link_dir}" ]; then + ${csudo}mkdir -p ${lib_link_dir} + log_print "${lib_link_dir} directory created" + fi #install log and data dir , then ln to /usr/local/taos ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} @@ -576,13 +647,27 @@ function install_TDengine() { ${csudo}ln -s ${log_dir} ${log_link_dir} || : ${csudo}ln -s ${data_dir} ${data_link_dir} || : +} + +function install_TDengine() { + echo -e "${GREEN}Start to install TDengine...${NC}" + log_print "start to install TDengine" + + checkDirectory # Install include, lib, binary and service - install_include - install_lib + install_include && + install_lib && install_bin + + if [[ "$?" != 0 ]];then + log_print "install TDengine failed!" + return 1 + fi + install_config install_taosadapter_config + install_taoskeeper_config install_taosadapter_service install_service install_app @@ -622,11 +707,14 @@ function install_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: taos -h $serverFqdn${GREEN_DARK} to login into TDengine server${NC}" echo fi + log_print "install TDengine successfully!" echo echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" } ## ==============================Main program starts from here============================ +${csudo}rm -rf ${install_log_path} +log_print "installer start..." serverFqdn=$(hostname) install_TDengine diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 8e5630ec2f7bc89d739486efdec7fc55acb2b046..9c50c4582db01fab94bb82f1556c3a94df741b1d 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -36,6 +36,16 @@ clientName="taos" uninstallScript="rmtaos" productName="TDengine" +serverName2="taosd" +clientName2="taos" +productName2="TDengine" + +benchmarkName2="${clientName2}Benchmark" +dumpName2="${clientName2}dump" +uninstallScript2="rm${clientName2}" + +installDir="/usr/local/${clientName}" + #install main path install_main_dir=${installDir} data_link_dir=${installDir}/data @@ -107,6 +117,15 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : + ${csudo}rm -f ${bin_link_dir}/taosx || : + + if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + fi } function clean_local_bin() { @@ -147,7 +166,7 @@ function clean_log() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/${taos_service_name}.service" if systemctl is-active --quiet ${taos_service_name}; then - echo "${productName} ${serverName} is running, stopping it..." + echo "${productName2} ${serverName2} is running, stopping it..." ${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null @@ -155,7 +174,7 @@ function clean_service_on_systemd() { taosadapter_service_config="${service_config_dir}/taosadapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then - echo "${productName} taosAdapter is running, stopping it..." + echo "${productName2} ${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -163,7 +182,7 @@ function clean_service_on_systemd() { tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" if systemctl is-active --quiet ${tarbitrator_service_name}; then - echo "${productName} tarbitrator is running, stopping it..." + echo "${productName2} tarbitrator is running, stopping it..." ${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null @@ -172,12 +191,12 @@ function clean_service_on_systemd() { function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - echo "${productName} ${serverName} is running, stopping it..." + echo "${productName2} ${serverName2} is running, stopping it..." ${csudo}service ${serverName} stop || : fi if ps aux | grep -v grep | grep tarbitrator &>/dev/null; then - echo "${productName} tarbitrator is running, stopping it..." + echo "${productName2} tarbitrator is running, stopping it..." ${csudo}service tarbitratord stop || : fi @@ -272,5 +291,5 @@ if [ "$osType" = "Darwin" ]; then ${csudo}rm -rf /Applications/TDengine.app fi -echo -e "${GREEN}${productName} is removed successfully!${NC}" +echo -e "${GREEN}${productName2} is removed successfully!${NC}" echo diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 7ab7fa85998284e0dcb4cd158eb8599806fd7470..10a0fb5e02a7fbc10b05765d4036e76b0bd1e4c8 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -12,6 +12,15 @@ installDir="/usr/local/taos" clientName="taos" uninstallScript="rmtaos" +clientName2="taos" +productName2="TDengine" + +benchmarkName2="${clientName}Benchmark" +dumpName2="${clientName}dump" +uninstallScript2="rm${clientName}" + +installDir="/usr/local/${clientName}" + #install main path install_main_dir=${installDir} @@ -40,6 +49,12 @@ function clean_bin() { ${csudo}rm -f ${bin_link_dir}/taosdump || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/set_core || : + + if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then + ${csudo}rm -f ${bin_link_dir}/${clientName2} || : + ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${uninstallScript2} || : + fi } function clean_lib() { @@ -82,5 +97,5 @@ clean_config ${csudo}rm -rf ${install_main_dir} -echo -e "${GREEN}TDengine client is removed successfully!${NC}" +echo -e "${GREEN}${productName2} client is removed successfully!${NC}" echo diff --git a/packaging/tools/tdengine.iss b/packaging/tools/tdengine.iss index 3a22c737743872528887616b83b3ec58329e8795..02f412f5d46fcc1ef2627c4dd6378775dd755995 100644 --- a/packaging/tools/tdengine.iss +++ b/packaging/tools/tdengine.iss @@ -18,7 +18,6 @@ #define MyAppDLLName "\driver\*.dll" ;#define MyAppVersion "3.0" ;#define MyAppInstallName "TDengine" -;#define MyAppInstallName "TDengine" [Setup] VersionInfoVersion={#MyAppVersion} AppId={{A0F7A93C-79C4-485D-B2B8-F0D03DF42FAB} @@ -59,11 +58,14 @@ Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igN Source: {#MyAppSourceDir}{#MyAppIncludeName}; DestDir: "{app}\include"; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppExeName}; DestDir: "{app}"; Excludes: {#MyAppExcludeSource} ; Flags: igNoreversion recursesubdirs createallsubdirs Source: {#MyAppSourceDir}{#MyAppTaosdemoExeName}; DestDir: "{app}"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\taos.exe; DestDir: "{app}"; DestName: "{#CusPrompt}.exe"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\taosBenchmark.exe; DestDir: "{app}"; DestName: "{#CusPrompt}Benchmark.exe"; Flags: igNoreversion recursesubdirs createallsubdirs +Source: {#MyAppSourceDir}\taosdump.exe; DestDir: "{app}"; DestName: "{#CusPrompt}dump.exe"; Flags: igNoreversion recursesubdirs createallsubdirs [run] -Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\TDengine\\taosd.exe --win_service""" ; Flags: runhidden -Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\TDengine\\taosadapter.exe""" ; Flags: runhidden +Filename: {sys}\sc.exe; Parameters: "create taosd start= DEMAND binPath= ""C:\\{#CusName}\\taosd.exe --win_service""" ; Flags: runhidden +Filename: {sys}\sc.exe; Parameters: "create taosadapter start= DEMAND binPath= ""C:\\{#CusName}\\taosadapter.exe""" ; Flags: runhidden [UninstallRun] RunOnceId: "stoptaosd"; Filename: {sys}\sc.exe; Parameters: "stop taosd" ; Flags: runhidden @@ -73,8 +75,8 @@ RunOnceId: "deltaosadapter"; Filename: {sys}\sc.exe; Parameters: "delete taosada [Registry] Root: HKLM; Subkey: "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"; \ - ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\TDengine"; \ - Check: NeedsAddPath('C:\TDengine') + ValueType: expandsz; ValueName: "Path"; ValueData: "{olddata};C:\{#CusName}"; \ + Check: NeedsAddPath('C:\{#CusName}') [Code] function NeedsAddPath(Param: string): boolean; @@ -104,11 +106,11 @@ Name: "desktopicon";Description: "{cm:CreateDesktopIcon}"; GroupDescription:"{cm [Icons] Name:"{group}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; IconFilename: "{app}\include\{#MyAppIco}" -Name:"{group}\Open TDengine Directory"; Filename: "{app}\" +Name:"{group}\Open {#CusName} Directory"; Filename: "{app}\" Name:"{group}\Taosdemo"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taosdemo.exe" ; IconFilename: "{app}\include\{#MyAppIco}" Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}" ; IconFilename: "{app}\include\{#MyAppIco}" Name:"{commondesktop}\Taos Shell"; Filename: "{app}\include\{#MyAppTaosExeName}" ; Parameters: "taos.exe" ; Tasks: desktopicon; WorkingDir: "{app}" ; IconFilename: "{app}\include\{#MyAppIco}" [Messages] -ConfirmUninstall=Do you really want to uninstall TDengine from your computer?%n%nPress [Y] to completely delete %1 and all its components;%nPress [N] to keep the software on your computer. +ConfirmUninstall=Do you really want to uninstall {#CusName} from your computer?%n%nPress [Y] to completely delete %1 and all its components;%nPress [N] to keep the software on your computer. diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index fb011ce3e4ae3ac3eeef3709fa397e2065e536a1..7cc7a1717a1e975e70552348027dbfb059756618 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -147,9 +147,12 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; - int8_t schemalessType; // todo remove it, this attribute should be move to request } STscObj; +typedef struct STscDbg { + bool memEnable; +} STscDbg; + typedef struct SResultColumn { union { char* nullbitmap; // bitmap, one bit for each item in the list diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index 0cb36ff61db570f2c6fb488d2964fc38c2cfd7f4..c29f495201d7b9ec0360abf6a5414798739b3da8 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -30,7 +30,7 @@ extern "C" { #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebugL(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLongString("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) -//#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscPerf(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", 0, cDebugFlag, __VA_ARGS__); }} while(0) // clang-format on #ifdef __cplusplus diff --git a/source/client/inc/clientSml.h b/source/client/inc/clientSml.h new file mode 100644 index 0000000000000000000000000000000000000000..3dcba673bfa843823561e565ddbe58586d82b774 --- /dev/null +++ b/source/client/inc/clientSml.h @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef TDENGINE_CLIENTSML_H +#define TDENGINE_CLIENTSML_H +#ifdef __cplusplus +extern "C" { +#endif + +#include "catalog.h" +#include "clientInt.h" +#include "osThread.h" +#include "query.h" +#include "taos.h" +#include "taoserror.h" +#include "tcommon.h" +#include "tdef.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmsg.h" +#include "tname.h" +#include "ttime.h" +#include "ttypes.h" +#include "cJSON.h" + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#ifndef likely +#define likely(expr) expect((expr) != 0, 1) +#endif +#ifndef unlikely +#define unlikely(expr) expect((expr) != 0, 0) +#endif + +#define SPACE ' ' +#define COMMA ',' +#define EQUAL '=' +#define QUOTE '"' +#define SLASH '\\' + +#define JUMP_SPACE(sql, sqlEnd) \ + while (sql < sqlEnd) { \ + if (unlikely(*sql == SPACE)) \ + sql++; \ + else \ + break; \ + } + +#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) +#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) + +#define TS "_ts" +#define TS_LEN 3 +#define VALUE "_value" +#define VALUE_LEN 6 + +#define OTD_JSON_FIELDS_NUM 4 +#define MAX_RETRY_TIMES 5 +typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; + +typedef enum { + SCHEMA_ACTION_NULL, + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, +} ESchemaAction; + +typedef struct { + const void *key; + int32_t keyLen; + void *value; + bool used; +}Node; + +typedef struct NodeList{ + Node data; + struct NodeList* next; +}NodeList; + +typedef struct { + char *measure; + char *tags; + char *cols; + char *timestamp; + char *measureTag; + + int32_t measureLen; + int32_t measureTagsLen; + int32_t tagsLen; + int32_t colsLen; + int32_t timestampLen; + + SArray *colArray; +} SSmlLineInfo; + +typedef struct { + const char *sTableName; // super table name + int32_t sTableNameLen; + char childTableName[TSDB_TABLE_NAME_LEN]; + uint64_t uid; +// void *key; // for openTsdb + + SArray *tags; + + // elements are SHashObj for find by key quickly + SArray *cols; + STableDataCxt *tableDataCtx; +} SSmlTableInfo; + +typedef struct { + SArray *tags; // save the origin order to create table + SHashObj *tagHash; // elements are + + SArray *cols; + SHashObj *colHash; + + STableMeta *tableMeta; +} SSmlSTableMeta; + +typedef struct { + int32_t len; + char *buf; +} SSmlMsgBuf; + +typedef struct { + int32_t code; + int32_t lineNum; + + int32_t numOfSTables; + int32_t numOfCTables; + int32_t numOfCreateSTables; + int32_t numOfAlterColSTables; + int32_t numOfAlterTagSTables; + + int64_t parseTime; + int64_t schemaTime; + int64_t insertBindTime; + int64_t insertRpcTime; + int64_t endTime; +} SSmlCostInfo; + +typedef struct { + int64_t id; + + SMLProtocolType protocol; + int8_t precision; + bool reRun; + bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) + bool isRawLine; + int32_t ttl; + int32_t uid; // used for automatic create child table + + SHashObj *childTables; + SHashObj *superTables; + SHashObj *pVgHash; + + STscObj *taos; + SCatalog *pCatalog; + SRequestObj *pRequest; + SQuery *pQuery; + + SSmlCostInfo cost; + int32_t lineNum; + SSmlMsgBuf msgBuf; + + cJSON *root; // for parse json + int8_t offset[OTD_JSON_FIELDS_NUM]; + SSmlLineInfo *lines; // element is SSmlLineInfo + bool parseJsonByLib; + SArray *tagJsonArray; + SArray *valueJsonArray; + + // + SArray *preLineTagKV; + SArray *maxTagKVs; + SArray *preLineColKV; + + SSmlLineInfo preLine; + STableMeta *currSTableMeta; + STableDataCxt *currTableDataCtx; + bool needModifySchema; +} SSmlHandle; + +#define IS_SAME_CHILD_TABLE (elements->measureTagsLen == info->preLine.measureTagsLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureTagsLen) == 0) + +#define IS_SAME_SUPER_TABLE (elements->measureLen == info->preLine.measureLen \ +&& memcmp(elements->measure, info->preLine.measure, elements->measureLen) == 0) + +#define IS_SAME_KEY (maxKV->keyLen == kv.keyLen && memcmp(maxKV->key, kv.key, kv.keyLen) == 0) + +extern int64_t smlFactorNS[3]; +extern int64_t smlFactorS[3]; + +typedef int32_t (*_equal_fn_sml)(const void *, const void *); + +SSmlHandle *smlBuildSmlInfo(TAOS *taos); +void smlDestroyInfo(SSmlHandle *info); +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset); +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset); +//SArray *smlJsonParseTags(char *start, char *end); +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg); +void* nodeListGet(NodeList* list, const void *key, int32_t len, _equal_fn_sml fn); +int nodeListSet(NodeList** list, const void *key, int32_t len, void* value, _equal_fn_sml fn); +int nodeListSize(NodeList* list); +bool smlDoubleToInt64OverFlow(double num); +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2); +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg); +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision); +int8_t smlGetTsTypeByLen(int32_t len); +SSmlTableInfo* smlBuildTableInfo(int numRows, const char* measure, int32_t measureLen); +SSmlSTableMeta* smlBuildSTableMeta(bool isDataFormat); +int32_t smlSetCTableName(SSmlTableInfo *oneTable); +STableMeta* smlGetMeta(SSmlHandle *info, const void* measure, int32_t measureLen); +int32_t is_same_child_table_telnet(const void *a, const void *b); +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len); +int32_t smlClearForRerun(SSmlHandle *info); +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg); +uint8_t smlGetTimestampLen(int64_t num); +void clearColValArray(SArray* pCols); +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag); + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements); +int32_t smlParseJSON(SSmlHandle *info, char *payload); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_CLIENTSML_H diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index e5507ccf816b9fd591e6e27ee2974badcb5734ed..2b42de93e3f45ccb95c085c31825d4e0a117538c 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -21,8 +21,6 @@ extern "C" { #endif #include "catalog.h" -typedef void STableDataBlocks; - typedef enum { STMT_TYPE_INSERT = 1, STMT_TYPE_MULTI_INSERT, @@ -43,8 +41,8 @@ typedef enum { } STMT_STATUS; typedef struct SStmtTableCache { - STableDataBlocks *pDataBlock; - void *boundTags; + STableDataCxt *pDataCtx; + void *boundTags; } SStmtTableCache; typedef struct SStmtQueryResInfo { @@ -71,10 +69,11 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { - int32_t affectedRows; - SRequestObj *pRequest; - SHashObj *pBlockHash; - bool autoCreateTbl; + int32_t affectedRows; + SRequestObj *pRequest; + SHashObj *pBlockHash; + STableDataCxt *pCurrBlock; + SSubmitTbData *pCurrTbData; } SStmtExecInfo; typedef struct SStmtSQLInfo { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 06ef7b7c9c702c3b3517efbfbf026a970305ea36..23486480414ed605049805ef44cbc6b03ddcc55e 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -23,16 +23,17 @@ #include "scheduler.h" #include "tcache.h" #include "tglobal.h" +#include "thttp.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "thttp.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 +STscDbg tscDbg = {0}; SAppInfo appInfo; int64_t lastClusterId = 0; int32_t clientReqRefPool = -1; @@ -65,7 +66,10 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { static void deregisterRequest(SRequestObj *pRequest) { const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable - assert(pRequest != NULL); + if (pRequest == NULL) { + tscError("pRequest == NULL"); + return; + } STscObj *pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; @@ -80,17 +84,18 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); if (pRequest->pQuery && pRequest->pQuery->pRoot) { - if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->pQuery->pRoot->type && (0 == ((SVnodeModifOpStmt*)pRequest->pQuery->pRoot)->sqlNodeType)) { + if (QUERY_NODE_VNODE_MODIFY_STMT == pRequest->pQuery->pRoot->type && + (0 == ((SVnodeModifyOpStmt *)pRequest->pQuery->pRoot)->sqlNodeType)) { tscDebug("insert duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 - "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", - duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, - pRequest->metric.planCostUs, pRequest->metric.execCostUs); + "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", + duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, + pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 - "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", - duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, - pRequest->metric.planCostUs, pRequest->metric.execCostUs); + "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", + duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, + pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } @@ -266,7 +271,6 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - pObj->schemalessType = 1; atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); @@ -369,9 +373,9 @@ void doDestroyRequest(void *p) { } if (pRequest->syncQuery) { - if (pRequest->body.param){ - tsem_destroy(&((SSyncQueryParam*)pRequest->body.param)->sem); - } + if (pRequest->body.param) { + tsem_destroy(&((SSyncQueryParam *)pRequest->body.param)->sem); + } taosMemoryFree(pRequest->body.param); } @@ -398,20 +402,20 @@ static void *tscCrashReportThreadFp(void *param) { setThreadName("client-crashReport"); char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s%s.taosCrashLog", tsLogDir, TD_DIRSEP); - char *pMsg = NULL; - int64_t msgLen = 0; + char *pMsg = NULL; + int64_t msgLen = 0; TdFilePtr pFile = NULL; - bool truncateFile = false; - int32_t sleepTime = 200; - int32_t reportPeriodNum = 3600 * 1000 / sleepTime; - int32_t loopTimes = reportPeriodNum; + bool truncateFile = false; + int32_t sleepTime = 200; + int32_t reportPeriodNum = 3600 * 1000 / sleepTime; + int32_t loopTimes = reportPeriodNum; #ifdef WINDOWS if (taosCheckCurrentInDll()) { atexit(crashReportThreadFuncUnexpectedStopped); } #endif - + while (1) { if (clientStop) break; if (loopTimes++ < reportPeriodNum) { @@ -441,12 +445,12 @@ static void *tscCrashReportThreadFp(void *param) { pMsg = NULL; continue; } - + if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); truncateFile = false; } - + taosMsleep(sleepTime); loopTimes = 0; } @@ -459,11 +463,11 @@ int32_t tscCrashReportInit() { if (!tsEnableCrashReport) { return 0; } - + TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - TdThread crashReportThread; + TdThread crashReportThread; if (taosThreadCreate(&crashReportThread, &thAttr, tscCrashReportThreadFp, NULL) != 0) { tscError("failed to create crashReport thread since %s", strerror(errno)); return -1; @@ -488,27 +492,37 @@ void tscStopCrashReport() { } } - void tscWriteCrashInfo(int signum, void *sigInfo, void *context) { - char *pMsg = NULL; + char *pMsg = NULL; const char *flags = "UTL FATAL "; ELogLevel level = DEBUG_FATAL; int32_t dflag = 255; - int64_t msgLen= -1; + int64_t msgLen = -1; if (tsEnableCrashReport) { if (taosGenCrashJsonMsg(signum, &pMsg, lastClusterId, appInfo.startTime)) { taosPrintLog(flags, level, dflag, "failed to generate crash json msg"); } else { - msgLen = strlen(pMsg); + msgLen = strlen(pMsg); } } taosLogCrashInfo("taos", pMsg, msgLen, signum, sigInfo); } - void taos_init_imp(void) { +#if defined(LINUX) + if (tscDbg.memEnable) { + int32_t code = taosMemoryDbgInit(); + if (code) { + printf("failed to init memory dbg, error:%s\n", tstrerror(code)); + } else { + tsAsyncLog = false; + printf("memory dbg enabled\n"); + } + } +#endif + // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. atexit(taos_cleanup); @@ -534,7 +548,8 @@ void taos_init_imp(void) { initQueryModuleMsgHandle(); if (taosConvInit() != 0) { - ASSERTS(0, "failed to init conv"); + tscError("failed to init conv"); + return; } rpcInit(); @@ -561,7 +576,7 @@ void taos_init_imp(void) { taosThreadMutexInit(&appInfo.mutex, NULL); tscCrashReportInit(); - + tscDebug("client is initialized successfully"); } @@ -612,6 +627,9 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("failed to set cfg:%s to %s since %s", pItem->name, str, terrstr()); } else { tscInfo("set cfg:%s to %s", pItem->name, str); + if (TSDB_OPTION_SHELL_ACTIVITY_TIMER == option || TSDB_OPTION_USE_ADAPTER == option) { + code = taosSetCfg(pCfg, pItem->name); + } } return code; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 5c4bcf794637143fdac038ef6ee7c011dac217fa..b5838386dbf94a3868c286cecc895df8162c33fd 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -403,7 +403,6 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.subPlanNum = 0; } desc.subPlanNum = taosArrayGetSize(desc.subDesc); - ASSERT(desc.subPlanNum == taosArrayGetSize(desc.subDesc)); } else { desc.subDesc = NULL; } @@ -789,7 +788,7 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.pData = buf; pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; - pInfo->param = strdup(pAppHbMgr->key); + pInfo->param = taosStrdup(pAppHbMgr->key); pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; @@ -843,7 +842,10 @@ static void hbStopThread() { } SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { - hbMgrInit(); + if(hbMgrInit() != 0){ + terrno = TSDB_CODE_TSC_INTERNAL_ERROR; + return NULL; + } SAppHbMgr *pAppHbMgr = taosMemoryMalloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -854,7 +856,7 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { pAppHbMgr->connKeyCnt = 0; pAppHbMgr->reportCnt = 0; pAppHbMgr->reportBytes = 0; - pAppHbMgr->key = strdup(key); + pAppHbMgr->key = taosStrdup(key); // init app info pAppHbMgr->pAppInstInfo = pAppInstInfo; @@ -929,16 +931,28 @@ int hbMgrInit() { TdThreadMutexAttr attr = {0}; int ret = taosThreadMutexAttrInit(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrInit error") + return ret; + } ret = taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrSetType error") + return ret; + } ret = taosThreadMutexInit(&clientHbMgr.lock, &attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexInit error") + return ret; + } ret = taosThreadMutexAttrDestroy(&attr); - assert(ret == 0); + if(ret != 0){ + uError("hbMgrInit:taosThreadMutexAttrDestroy error") + return ret; + } // init handle funcs hbMgrInitHandle(); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 079edcd6671bfc1afabd2aaabe1f6c03c48dbff7..a6d2fad816f8149cd27756c9356fed770239e1e1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -52,7 +52,7 @@ static bool validateDbName(const char* db) { return stringLengthCheck(db, TSDB_D static char* getClusterKey(const char* user, const char* auth, const char* ip, int32_t port) { char key[512] = {0}; snprintf(key, sizeof(key), "%s:%s:%s:%d", user, auth, ip, port); - return strdup(key); + return taosStrdup(key); } bool chkRequestKilled(void* param) { @@ -252,7 +252,6 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .svrVer = pTscObj->sVer, @@ -286,7 +285,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC int32_t execLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { SRetrieveTableRsp* pRsp = NULL; - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -324,7 +323,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { return; } - int32_t code = qExecCommand(pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); + int32_t code = qExecCommand(&pRequest->pTscObj->id, pRequest->pTscObj->sysInfo, pQuery->pRoot, &pRsp); if (TSDB_CODE_SUCCESS == code && NULL != pRsp) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRsp, false, true); } @@ -466,7 +465,10 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra } void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols) { - ASSERT(pSchema != NULL && numOfCols > 0); + if (pResInfo == NULL || pSchema == NULL || numOfCols <= 0) { + tscError("invalid paras, pResInfo == NULL || pSchema == NULL || numOfCols <= 0"); + return; + } pResInfo->numOfCols = numOfCols; if (pResInfo->fields != NULL) { @@ -477,7 +479,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); - ASSERT(numOfCols == pResInfo->numOfCols); + if (numOfCols != pResInfo->numOfCols) { + tscError("numOfCols:%d != pResInfo->numOfCols:%d", numOfCols, pResInfo->numOfCols); + return; + } for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { pResInfo->fields[i].bytes = pSchema[i].bytes; @@ -754,47 +759,21 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { - int32_t code = 0; - SArray* pArray = NULL; - SSubmitRsp* pRsp = (SSubmitRsp*)res; - if (pRsp->nBlocks <= 0) { - taosMemoryFreeClear(pRsp->pBlocks); + SArray* pArray = NULL; + SSubmitRsp2* pRsp = (SSubmitRsp2*)res; + if (NULL == pRsp->aCreateTbRsp) { return TSDB_CODE_SUCCESS; } - pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pRsp->pBlocks + i; - if (blk->pMeta) { - handleCreateTbExecRes(blk->pMeta, pCatalog); - tFreeSTableMetaRsp(blk->pMeta); - taosMemoryFreeClear(blk->pMeta); - } - - if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { - continue; + int32_t tbNum = taosArrayGetSize(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < tbNum; ++i) { + SVCreateTbRsp* pTbRsp = (SVCreateTbRsp*)taosArrayGet(pRsp->aCreateTbRsp, i); + if (pTbRsp->pMeta) { + handleCreateTbExecRes(pTbRsp->pMeta, pCatalog); } - - STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; - taosArrayPush(pArray, &tbSver); } - SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = *epset}; - - code = catalogChkTbMetaVersion(pCatalog, &conn, pArray); - -_return: - - taosArrayDestroy(pArray); - return code; + return TSDB_CODE_SUCCESS; } int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet* epset) { @@ -895,7 +874,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { } static bool incompletaFileParsing(SNode* pStmt) { - return QUERY_NODE_VNODE_MODIF_STMT != nodeType(pStmt) ? false : ((SVnodeModifOpStmt*)pStmt)->fileProcessing; + return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing; } // todo refacto the error code mgmt @@ -974,7 +953,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue if (pQuery->pRoot && !pRequest->inRetry) { STscObj* pTscObj = pRequest->pTscObj; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; - if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type) { + if (QUERY_NODE_VNODE_MODIFY_STMT == pQuery->pRoot->type) { atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertsReq, 1); } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { atomic_add_fetch_64((int64_t*)&pActivity->numOfQueryReq, 1); @@ -1078,7 +1057,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; - if (QUERY_NODE_VNODE_MODIF_STMT != nodeType(pQuery->pRoot)) { + if (QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pQuery->pRoot)) { buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); } @@ -1126,7 +1105,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM if (pQuery->pRoot && !pRequest->inRetry) { STscObj* pTscObj = pRequest->pTscObj; SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; - if (QUERY_NODE_VNODE_MODIF_STMT == pQuery->pRoot->type && (0 == ((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType)) { + if (QUERY_NODE_VNODE_MODIFY_STMT == pQuery->pRoot->type && + (0 == ((SVnodeModifyOpStmt*)pQuery->pRoot)->sqlNodeType)) { atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertsReq, 1); } else if (QUERY_NODE_SELECT_STMT == pQuery->pRoot->type) { atomic_add_fetch_64((int64_t*)&pActivity->numOfQueryReq, 1); @@ -1388,7 +1368,10 @@ int32_t doProcessMsgFromServer(void* param) { SEpSet* pEpSet = arg->pEpset; SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->info.ahandle; - assert(pMsg->info.ahandle != NULL); + if (pMsg->info.ahandle == NULL) { + tscError("doProcessMsgFromServer pMsg->info.ahandle == NULL"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } STscObj* pTscObj = NULL; STraceId* trace = &pMsg->info.traceId; @@ -1401,7 +1384,11 @@ int32_t doProcessMsgFromServer(void* param) { if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); if (pRequest) { - assert(pRequest->self == pSendInfo->requestObjRefId); + if (pRequest->self != pSendInfo->requestObjRefId) { + tscError("doProcessMsgFromServer pRequest->self:%" PRId64 " != pSendInfo->requestObjRefId:%" PRId64, + pRequest->self, pSendInfo->requestObjRefId); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pTscObj = pRequest->pTscObj; } } @@ -1534,7 +1521,9 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo) { } void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if (pRequest == NULL) { + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1588,7 +1577,9 @@ static void syncFetchFn(void* param, TAOS_RES* res, int32_t numOfRows) { } void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) { - assert(pRequest != NULL); + if (pRequest == NULL) { + return NULL; + } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { @@ -1652,8 +1643,13 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int char* pStart = pCol->offset[j] + pCol->pData; int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); - ASSERT(len <= bytes); - ASSERT((p + len) < (pResultInfo->convertBuf[i] + colLength[i])); + if (len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { + tscError( + "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " + "colLength[i]):%p", + len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } varDataSetLen(p, len); pCol->offset[j] = (p - pResultInfo->convertBuf[i]); @@ -1670,9 +1666,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int } int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols) { - int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); - ASSERT(numOfCols == cols); - return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) * 3 + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)); } @@ -1682,6 +1675,12 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column // length | + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if (ASSERT(numOfCols == cols)) { + tscError("estimateJsonLen error: numOfCols:%d != cols:%d", numOfCols, cols); + return -1; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1715,7 +1714,8 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { len += (VARSTR_HEADER_SIZE + 5); } else { - ASSERT(0); + tscError("estimateJsonLen error: invalid type:%d", jsonInnerType); + return -1; } } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1749,12 +1749,21 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); + if (dataLen <= 0) { + return TSDB_CODE_TSC_INTERNAL_ERROR; + } pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t totalLen = 0; + int32_t cols = *(int32_t*)(p + sizeof(int32_t) * 3); + if (ASSERT(numOfCols == cols)) { + tscError("doConvertJson error: numOfCols:%d != cols:%d", numOfCols, cols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } + int32_t len = getVersion1BlockMetaSize(p, numOfCols); memcpy(p1, p, len); @@ -1775,8 +1784,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int for (int32_t i = 0; i < numOfCols; ++i) { int32_t colLen = htonl(colLength[i]); int32_t colLen1 = htonl(colLength1[i]); - ASSERT(colLen < dataLen); - + if (ASSERT(colLen < dataLen)) { + tscError("doConvertJson error: colLen:%d >= dataLen:%d", colLen, dataLen); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; int32_t* offset1 = (int32_t*)pStart1; @@ -1821,7 +1832,8 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); } else { - ASSERT(0); + tscError("doConvertJson error: invalid type:%d", jsonInnerType); + return TSDB_CODE_TSC_INTERNAL_ERROR; } offset1[j] = len; @@ -1859,7 +1871,10 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows, bool convertUcs4) { - assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if (ASSERT(numOfCols > 0 && pFields != NULL && pResultInfo != NULL)) { + tscError("setResultDataPtr paras error"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (numOfRows == 0) { return TSDB_CODE_SUCCESS; } @@ -1888,7 +1903,11 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 int32_t cols = *(int32_t*)p; p += sizeof(int32_t); - ASSERT(rows == numOfRows && cols == numOfCols); + if (ASSERT(rows == numOfRows && cols == numOfCols)) { + tscError("setResultDataPtr paras error:rows;%d numOfRows:%d cols:%d numOfCols:%d", rows, numOfRows, cols, + numOfCols); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } int32_t hasColumnSeg = *(int32_t*)p; p += sizeof(int32_t); @@ -1915,7 +1934,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 colLength[i] = htonl(colLength[i]); if (colLength[i] >= dataLen) { tscError("invalid colLength %d, dataLen %d", colLength[i], dataLen); - ASSERT(0); + return TSDB_CODE_TSC_INTERNAL_ERROR; } if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { @@ -1953,7 +1972,11 @@ char* getDbOfConnection(STscObj* pObj) { } void setConnectionDB(STscObj* pTscObj, const char* db) { - assert(db != NULL && pTscObj != NULL); + if (db == NULL || pTscObj == NULL) { + tscError("setConnectionDB para is NULL"); + return; + } + taosThreadMutexLock(&pTscObj->mutex); tstrncpy(pTscObj->db, db, tListLen(pTscObj->db)); taosThreadMutexUnlock(&pTscObj->mutex); @@ -1971,7 +1994,10 @@ void resetConnectDB(STscObj* pTscObj) { int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, bool freeAfterUse) { - assert(pResultInfo != NULL && pRsp != NULL); + if (pResultInfo == NULL || pRsp == NULL) { + tscError("setQueryResultFromRsp paras is null"); + return TSDB_CODE_TSC_INTERNAL_ERROR; + } if (freeAfterUse) taosMemoryFreeClear(pResultInfo->pRspMsg); @@ -2322,6 +2348,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { return NULL; } + tscDebug("taos_query start with sql:%s", sql); + SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); @@ -2336,6 +2364,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { taosMemoryFree(param); } + tscDebug("taos_query end with sql:%s", sql); + return pRequest; } diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index 859d4ec80ff6e954259415dabaf1761eb3e7129e..750ba684f42c2f35cd6f5180e6d1d722ad3e0e16 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -574,7 +574,11 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI TAOS_RES *tres = (TAOS_RES *)res; int32_t numOfFields = taos_num_fields(tres); - assert(numOfFields > 0); + if(numOfFields <= 0){ + jniError("jobj:%p, conn:%p, query interrupted. taos_num_fields error code:%d, msg:%s", jobj, tscon, numOfFields, + taos_errstr(tres)); + return JNI_RESULT_SET_NULL; + } void *data; int32_t numOfRows; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 53efff023db587cdf565b4a3dadc9acb54c03533..4ba51ce50dfc6dd74ade07d5a004daf5034e15d8 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -293,7 +293,6 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { tscError("invalid result passed to taos_fetch_row"); return NULL; } - return NULL; } int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { @@ -357,9 +356,13 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - assert(charLen <= fields[i].bytes && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){ + tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } else { - assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){ + tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); + } } memcpy(str + len, row[i], charLen); @@ -432,6 +435,22 @@ const char *taos_data_type(int type) { return "TSDB_DATA_TYPE_NCHAR"; case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON"; + case TSDB_DATA_TYPE_UTINYINT: + return "TSDB_DATA_TYPE_UTINYINT"; + case TSDB_DATA_TYPE_USMALLINT: + return "TSDB_DATA_TYPE_USMALLINT"; + case TSDB_DATA_TYPE_UINT: + return "TSDB_DATA_TYPE_UINT"; + case TSDB_DATA_TYPE_UBIGINT: + return "TSDB_DATA_TYPE_UBIGINT"; + case TSDB_DATA_TYPE_VARBINARY: + return "TSDB_DATA_TYPE_VARBINARY"; + case TSDB_DATA_TYPE_DECIMAL: + return "TSDB_DATA_TYPE_DECIMAL"; + case TSDB_DATA_TYPE_BLOB: + return "TSDB_DATA_TYPE_BLOB"; + case TSDB_DATA_TYPE_MEDIUMBLOB: + return "TSDB_DATA_TYPE_MEDIUMBLOB"; default: return "UNKNOWN"; } @@ -578,7 +597,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { (*numOfRows) = pResultInfo->numOfRows; return 0; } else { - ASSERT(0); + tscError("taos_fetch_block_s invalid res type"); return -1; } } @@ -671,6 +690,32 @@ const char *taos_get_server_info(TAOS *taos) { return pTscObj->sDetailVer; } +int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + if (pTscObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return -1; + } + + int code = TSDB_CODE_SUCCESS; + taosThreadMutexLock(&pTscObj->mutex); + if(database == NULL || len <= 0){ + if(required != NULL) *required = strlen(pTscObj->db) + 1; + terrno = TSDB_CODE_INVALID_PARA; + code = -1; + }else if(len < strlen(pTscObj->db) + 1){ + tstrncpy(database, pTscObj->db, len); + if(required) *required = strlen(pTscObj->db) + 1; + terrno = TSDB_CODE_INVALID_PARA; + code = -1; + }else{ + strcpy(database, pTscObj->db); + code = 0; + } + taosThreadMutexUnlock(&pTscObj->mutex); + return code; +} + static void destoryTablesReq(void *p) { STablesReq *pRes = (STablesReq *)p; taosArrayDestroy(pRes->pTables); @@ -800,7 +845,7 @@ static void doAsyncQueryFromParse(SMetaData *pResultMeta, void *param, int32_t c tstrerror(code)); if (code == TSDB_CODE_SUCCESS) { - //pWrapper->pCatalogReq->forceUpdate = false; + // pWrapper->pCatalogReq->forceUpdate = false; code = qContinueParseSql(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); } @@ -829,14 +874,16 @@ void continueInsertFromCsv(SSqlCallbackWrapper *pWrapper, SRequestObj *pRequest) tstrerror(code), pWrapper->pRequest->requestId); destorySqlCallbackWrapper(pWrapper); terrno = code; - pWrapper->pRequest->code = code; - pWrapper->pRequest->body.queryFp(pWrapper->pRequest->body.param, pWrapper->pRequest, code); + pRequest->code = code; + pRequest->body.queryFp(pRequest->body.param, pRequest, code); } } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { int64_t connId = *(int64_t *)taos; + tscDebug("taos_query_a start with sql:%s", sql); taosAsyncQueryImpl(connId, sql, fp, param, false); + tscDebug("taos_query_a end with sql:%s", sql); } void taos_query_a_with_reqid(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, int64_t reqid) { @@ -864,7 +911,6 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = NULL, .pUser = pTscObj->user, - .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .enableSysInfo = pTscObj->sysInfo, .async = true, @@ -990,8 +1036,14 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; pRequest->body.fetchFp = fp; @@ -1034,9 +1086,14 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - ASSERT(res != NULL && fp != NULL); - ASSERT(TD_RES_QUERY(res)); - + if(ASSERT(res != NULL && fp != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return; + } SRequestObj *pRequest = res; SReqResultInfo *pResultInfo = &pRequest->body.resInfo; @@ -1048,8 +1105,14 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - ASSERT(res != NULL); - ASSERT(TD_RES_QUERY(res)); + if(ASSERT(res != NULL)){ + tscError("taos_fetch_rows_a invalid paras"); + return NULL; + } + if(ASSERT(TD_RES_QUERY(res))){ + tscError("taos_fetch_rows_a res is NULL"); + return NULL; + } SRequestObj *pRequest = res; return pRequest->body.resInfo.pData; @@ -1369,8 +1432,7 @@ int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fiel } // let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields` -void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields) -{ +void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields) { (void)stmt; if (!fields) return; taosMemoryFree(fields); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 92d03103d723f30f9205bebeaa6b69de18286d4b..a5f963a9569d154167b4cc58818ae7f01edcf304 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -150,7 +150,6 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { pMsgSendInfo->msgType = pRequest->type; pMsgSendInfo->target.type = TARGET_TYPE_MNODE; - assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; pMsgSendInfo->fp = getMsgRspHandle(pRequest->type); return pMsgSendInfo; @@ -290,7 +289,9 @@ int32_t processUseDbRsp(void* param, SDataBuf* pMsg, int32_t code) { } int32_t processCreateSTableRsp(void* param, SDataBuf* pMsg, int32_t code) { - assert(pMsg != NULL && param != NULL); + if(pMsg == NULL || param == NULL){ + return TSDB_CODE_TSC_INVALID_INPUT; + } SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { @@ -471,9 +472,13 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); - blockDataDestroy(pBlock); + + if(len != rspSize - sizeof(SRetrieveTableRsp)){ + uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t) (rspSize - sizeof(SRetrieveTableRsp))); + return TSDB_CODE_TSC_INVALID_INPUT; + } + return TSDB_CODE_SUCCESS; } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 8c91eacf1eefcdbfffb9ce72a7dec80a86e92e3e..a12f94cf1d88f3f52c958e20dfa130f98813da70 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -183,7 +183,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { } string = cJSON_PrintUnformatted(json); - end: +end: cJSON_Delete(json); tFreeSMAltertbReq(&req); return string; @@ -204,7 +204,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { } string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); - _err: +_err: tDecoderClear(&coder); return string; } @@ -224,7 +224,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { } string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); - _err: +_err: tDecoderClear(&coder); return string; } @@ -306,7 +306,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON_AddItemToArray(tags, tag); } - end: +end: cJSON_AddItemToObject(json, "tags", tags); taosArrayDestroy(pTagVals); } @@ -364,7 +364,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } } - _exit: +_exit: for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; taosMemoryFreeClear(pCreateReq->comment); @@ -377,7 +377,10 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } static char* processAutoCreateTable(STaosxRsp* rsp) { - ASSERT(rsp->createTableNum != 0); + if (rsp->createTableNum <= 0) { + uError("WriteRaw:processAutoCreateTable rsp->createTableNum <= 0"); + goto _exit; + } SDecoder* decoder = taosMemoryCalloc(rsp->createTableNum, sizeof(SDecoder)); SVCreateTbReq* pCreateReq = taosMemoryCalloc(rsp->createTableNum, sizeof(SVCreateTbReq)); @@ -393,11 +396,14 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { goto _exit; } - ASSERT(pCreateReq[iReq].type == TSDB_CHILD_TABLE); + if (pCreateReq[iReq].type != TSDB_CHILD_TABLE) { + uError("WriteRaw:processAutoCreateTable pCreateReq[iReq].type != TSDB_CHILD_TABLE"); + goto _exit; + } } string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); - _exit: +_exit: for (int i = 0; i < rsp->createTableNum; i++) { tDecoderClear(&decoder[i]); taosMemoryFreeClear(pCreateReq[i].comment); @@ -498,7 +504,10 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { char* buf = NULL; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { - ASSERT(tTagIsJson(vAlterTbReq.pTagVal) == true); + if (!tTagIsJson(vAlterTbReq.pTagVal)) { + uError("processAlterTable isJson false"); + goto _exit; + } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 1, 1); @@ -519,7 +528,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } string = cJSON_PrintUnformatted(json); - _exit: +_exit: cJSON_Delete(json); tDecoderClear(&decoder); return string; @@ -552,12 +561,12 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { string = cJSON_PrintUnformatted(json); - _exit: +_exit: cJSON_Delete(json); tDecoderClear(&decoder); return string; } -static char* processDeleteTable(SMqMetaRsp* metaRsp){ +static char* processDeleteTable(SMqMetaRsp* metaRsp) { SDeleteRes req = {0}; SDecoder coder = {0}; int32_t code = TSDB_CODE_SUCCESS; @@ -591,7 +600,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: cJSON_Delete(json); tDecoderClear(&coder); return string; @@ -633,7 +642,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { string = cJSON_PrintUnformatted(json); - _exit: +_exit: cJSON_Delete(json); tDecoderClear(&decoder); return string; @@ -721,7 +730,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); tDecoderClear(&coder); @@ -791,7 +800,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tDecoderClear(&coder); return code; @@ -852,9 +861,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table @@ -935,7 +944,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; - end: +end: for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; taosMemoryFreeClear(pCreateReq->comment); @@ -1005,9 +1014,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -1060,7 +1069,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -1123,12 +1132,12 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { TAOS_RES* res = taos_query(taos, sql); SRequestObj* pRequest = (SRequestObj*)res; code = pRequest->code; - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_PAR_GET_META_ERROR) { code = TSDB_CODE_SUCCESS; } taos_free_result(res); - end: +end: tDecoderClear(&coder); return code; } @@ -1175,9 +1184,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { } SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1236,7 +1245,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { code = handleAlterTbExecRes(pRes->res, pCatalog); } } - end: +end: taosArrayDestroy(pArray); if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); @@ -1246,21 +1255,12 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -typedef struct { - SVgroupInfo vg; - void* data; -} VgData; - -static void destroyVgHash(void* data) { - VgData* vgData = (VgData*)data; - taosMemoryFreeClear(vgData->data); -} - -int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD *fields, int numFields){ +int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields, + int numFields) { int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1305,157 +1305,34 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numFields); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numFields; - - SResultColumn* pCol = taosMemoryCalloc(numFields, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numFields; ++i) { - if (IS_VAR_DATA_TYPE(fields[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int i = 0; i < numFields; i++) { - TAOS_FIELD* schema = &fields[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { // add none - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - }else{ - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[*index].offset[j] != -1) { - char* data = pCol[*index].pData + pCol[*index].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[*index].nullbitmap, j)) { - char* data = pCol[*index].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosHashCleanup(schemaHash); - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, fields, numFields, false); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); destroyRequest(pRequest); - taosMemoryFree(subReq); + taosHashCleanup(pVgHash); return code; } @@ -1463,7 +1340,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) int32_t code = TSDB_CODE_SUCCESS; STableMeta* pTableMeta = NULL; SQuery* pQuery = NULL; - SSubmitReq* subReq = NULL; + SHashObj* pVgHash = NULL; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); if (!pRequest) { @@ -1508,146 +1385,34 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) uError("WriteRaw:catalogGetTableMeta failed. table name: %s", tbname); goto end; } - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int32_t numOfCols = pTableMeta->tableInfo.numOfColumns; - - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < numOfCols; i++) { - SSchema* schema = pTableMeta->schema + i; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } - } - fLen -= sizeof(TSKEY); - - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(numOfCols - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - subReq = taosMemoryCalloc(1, totalLen); - SSubmitBlk* blk = POINTER_SHIFT(subReq, sizeof(SSubmitReq)); - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTableMeta->sversion); - tdSRowSetTpInfo(&rb, numOfCols, fLen); - int32_t dataLen = 0; - - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | - char* pStart = pData + getVersion1BlockMetaSize(pData, numOfCols); - int32_t* colLength = (int32_t*)pStart; - pStart += sizeof(int32_t) * numOfCols; - - SResultColumn* pCol = taosMemoryCalloc(numOfCols, sizeof(SResultColumn)); - - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE(pTableMeta->schema[i].type)) { - pCol[i].offset = (int32_t*)pStart; - pStart += rows * sizeof(int32_t); - } else { - pCol[i].nullbitmap = pStart; - pStart += BitmapLen(rows); - } - - pCol[i].pData = pStart; - pStart += colLength[i]; - } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - int32_t offset = 0; - for (int32_t k = 0; k < numOfCols; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - - if (IS_VAR_DATA_TYPE(pColumn->type)) { - if (pCol[k].offset[j] != -1) { - char* data = pCol[k].pData + pCol[k].offset[j]; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } else { - if (!colDataIsNull_f(pCol[k].nullbitmap, j)) { - char* data = pCol[k].pData + pColumn->bytes * j; - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, offset, k); - } else { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } - } - - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - - taosMemoryFree(pCol); - - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(pTableMeta->sversion); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(dataLen); - subReq->length = sizeof(SSubmitReq) + sizeof(SSubmitBlk) + schemaLen + dataLen; - subReq->numOfBlocks = 1; - - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); + pQuery = smlInitHandle(); + if (pQuery == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(subReq); goto end; } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + taosHashPut(pVgHash, (const char*)&vgData.vgId, sizeof(vgData.vgId), (char*)&vgData, sizeof(vgData)); + + code = rawBlockBindData(pQuery, pTableMeta, pData, NULL, NULL, 0, false); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - nodeStmt->pDataBlocks = taosArrayInit(1, POINTER_BYTES); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); + return code; } - dst->vg = vgData; - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - subReq = NULL; // no need free - taosArrayPush(nodeStmt->pDataBlocks, &dst); launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); destroyRequest(pRequest); - taosMemoryFree(subReq); + taosHashCleanup(pVgHash); return code; } @@ -1684,8 +1449,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1699,6 +1462,12 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1706,14 +1475,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1727,13 +1488,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); @@ -1745,173 +1499,47 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - fLen -= sizeof(TSKEY); - - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - int32_t schemaLen = 0; - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); + if (fields == NULL) { + goto end; + } for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + fields[i].type = pSW->pSchema[i].type; + fields[i].bytes = pSW->pSchema[i].bytes; + tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, NULL, fields, pSW->nCols, true); + taosMemoryFree(fields); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); + goto end; } - - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); } - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); - } launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: tDeleteSMqDataRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1920,7 +1548,6 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { return code; } - static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { int32_t code = TSDB_CODE_SUCCESS; SHashObj* pVgHash = NULL; @@ -1928,7 +1555,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) SMqTaosxRspObj rspObj = {0}; SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; - void* schemaContent = NULL; + SVCreateTbReq* pCreateReqDst = NULL; terrno = TSDB_CODE_SUCCESS; SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0); @@ -1955,8 +1582,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pVgHash, destroyVgHash); struct SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { @@ -1970,6 +1595,13 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) conn.requestObjRefId = pRequest->self; conn.mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + pQuery = smlInitHandle(); + if (pQuery == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + uDebug("raw data block num:%d\n", rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); @@ -1977,14 +1609,6 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) uError("WriteRaw:no schema, iter:%d", rspObj.resIter); goto end; } - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); - setResSchemaInfo(&rspObj.resInfo, pSW->pSchema, pSW->nCols); - - code = setQueryResultFromRsp(&rspObj.resInfo, pRetrieve, false, false); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw: setQueryResultFromRsp error"); - goto end; - } const char* tbName = (const char*)taosArrayGetP(rspObj.rsp.blockTbName, rspObj.resIter); if (!tbName) { @@ -1998,68 +1622,40 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); - VgData vgData = {0}; - code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &(vgData.vg)); - if (code != TSDB_CODE_SUCCESS) { - uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); - goto end; - } - // find schema data info - int32_t schemaLen = 0; - void* schemaData = NULL; for (int j = 0; j < rspObj.rsp.createTableNum; j++) { void** dataTmp = taosArrayGet(rspObj.rsp.createTableReq, j); int32_t* lenTmp = taosArrayGet(rspObj.rsp.createTableLen, j); SDecoder decoderTmp = {0}; SVCreateTbReq pCreateReq = {0}; + tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); + if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { + tDecoderClear(&decoderTmp); + uError("WriteRaw: tDecodeSVCreateTbReq error"); + code = TSDB_CODE_TMQ_INVALID_MSG; + goto end; + } - do{ - tDecoderInit(&decoderTmp, *dataTmp, *lenTmp); - if (tDecodeSVCreateTbReq(&decoderTmp, &pCreateReq) < 0) { - code = TSDB_CODE_MSG_DECODE_ERROR; - break; - } - - if (strcmp(tbName, pCreateReq.name) != 0) { - break; - } - - pCreateReq.ctb.suid = processSuid(pCreateReq.ctb.suid, pRequest->pDb); - - int32_t len = 0; - tEncodeSize(tEncodeSVCreateTbReq, &pCreateReq, len, code); - if(code != 0) { - code = TSDB_CODE_MSG_ENCODE_ERROR; - break; - } - taosMemoryFree(schemaContent); - schemaContent = taosMemoryMalloc(len); - if(!schemaContent) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; - } - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaContent, len); - code = tEncodeSVCreateTbReq(&encoder, &pCreateReq); - if (code != 0) { - tEncoderClear(&encoder); - code = TSDB_CODE_MSG_ENCODE_ERROR; - break; - } - schemaLen = len; - schemaData = schemaContent; - strcpy(pName.tname, pCreateReq.ctb.stbName); - tEncoderClear(&encoder); - }while(0); + if (pCreateReq.type != TSDB_CHILD_TABLE) { + uError("WriteRaw:pCreateReq.type != TSDB_CHILD_TABLE. table name: %s", tbName); + code = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + if (strcmp(tbName, pCreateReq.name) == 0) { + cloneSVreateTbReq(&pCreateReq, &pCreateReqDst); + pCreateReqDst->ctb.suid = processSuid(pCreateReqDst->ctb.suid, pRequest->pDb); + tDecoderClear(&decoderTmp); + break; + } tDecoderClear(&decoderTmp); - taosMemoryFreeClear(pCreateReq.comment); - taosArrayDestroy(pCreateReq.ctb.tagName); - if(code != 0) goto end; - if(schemaLen != 0) break; } + if (pCreateReqDst) { + strcpy(pName.tname, pCreateReqDst->ctb.stbName); + } else { + strcpy(pName.tname, tbName); + } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); @@ -2071,182 +1667,61 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uint16_t fLen = 0; - int32_t rowSize = 0; - int16_t nVar = 0; - for (int i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - SSchema* schema = &pTableMeta->schema[i]; - fLen += TYPE_BYTES[schema->type]; - rowSize += schema->bytes; - if (IS_VAR_DATA_TYPE(schema->type)) { - nVar++; - } + SVgroupInfo vg; + code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &vg); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:catalogGetTableHashVgroup failed. table name: %s", tbName); + goto end; } - fLen -= sizeof(TSKEY); - int32_t rows = rspObj.resInfo.numOfRows; - int32_t extendedRowSize = rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + nVar * sizeof(VarDataOffsetT) + - (int32_t)TD_BITMAP_BYTES(pTableMeta->tableInfo.numOfColumns - 1); - - int32_t submitLen = sizeof(SSubmitBlk) + schemaLen + rows * extendedRowSize; - - SSubmitReq* subReq = NULL; - SSubmitBlk* blk = NULL; - void* hData = taosHashGet(pVgHash, &vgData.vg.vgId, sizeof(vgData.vg.vgId)); - if (hData) { - vgData = *(VgData*)hData; - - int32_t totalLen = ((SSubmitReq*)(vgData.data))->length + submitLen; - void* tmp = taosMemoryRealloc(vgData.data, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - ((VgData*)hData)->data = tmp; - subReq = (SSubmitReq*)(vgData.data); - blk = POINTER_SHIFT(vgData.data, subReq->length); - } else { - int32_t totalLen = sizeof(SSubmitReq) + submitLen; - void* tmp = taosMemoryCalloc(1, totalLen); - if (tmp == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - vgData.data = tmp; - taosHashPut(pVgHash, (const char*)&vgData.vg.vgId, sizeof(vgData.vg.vgId), (char*)&vgData, sizeof(vgData)); - subReq = (SSubmitReq*)(vgData.data); - subReq->length = sizeof(SSubmitReq); - subReq->numOfBlocks = 0; - - blk = POINTER_SHIFT(vgData.data, sizeof(SSubmitReq)); + if (pCreateReqDst) { + pTableMeta->vgId = vg.vgId; + pTableMeta->uid = pCreateReqDst->uid; } - - // pSW->pSchema should be same as pTableMeta->schema - // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); - uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); - uint64_t uid = pTableMeta->uid; - int16_t sver = pTableMeta->sversion; - - void* blkSchema = POINTER_SHIFT(blk, sizeof(SSubmitBlk)); - if (schemaData) { - memcpy(blkSchema, schemaData, schemaLen); + void* hData = taosHashGet(pVgHash, &vg.vgId, sizeof(vg.vgId)); + if (hData == NULL) { + taosHashPut(pVgHash, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)); } - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - - SRowBuilder rb = {0}; - tdSRowInit(&rb, sver); - tdSRowSetTpInfo(&rb, pTableMeta->tableInfo.numOfColumns, fLen); - int32_t totalLen = 0; - SHashObj* schemaHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(rspObj.rsp.blockSchema, rspObj.resIter); + TAOS_FIELD* fields = taosMemoryCalloc(pSW->nCols, sizeof(TAOS_FIELD)); + if (fields == NULL) { + goto end; + } for (int i = 0; i < pSW->nCols; i++) { - SSchema* schema = &pSW->pSchema[i]; - taosHashPut(schemaHash, schema->name, strlen(schema->name), &i, sizeof(int32_t)); + fields[i].type = pSW->pSchema[i].type; + fields[i].bytes = pSW->pSchema[i].bytes; + tstrncpy(fields[i].name, pSW->pSchema[i].name, tListLen(pSW->pSchema[i].name)); } - - for (int32_t j = 0; j < rows; j++) { - tdSRowResetBuf(&rb, rowData); - - doSetOneRowPtr(&rspObj.resInfo); - rspObj.resInfo.current += 1; - - int32_t offset = 0; - for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { - const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if (!index) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NONE, NULL, false, offset, k); - } else { - char* colData = rspObj.resInfo.row[*index]; - if (!colData) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - } else { - if (IS_VAR_DATA_TYPE(pColumn->type)) { - colData -= VARSTR_HEADER_SIZE; - } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, offset, k); - } - } - if (pColumn->colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - offset += TYPE_BYTES[pColumn->type]; - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - totalLen += rowLen; + code = rawBlockBindData(pQuery, pTableMeta, pRetrieve->data, pCreateReqDst, fields, pSW->nCols, true); + taosMemoryFree(fields); + if (code != TSDB_CODE_SUCCESS) { + uError("WriteRaw:rawBlockBindData failed"); + goto end; } - - taosHashCleanup(schemaHash); - blk->uid = htobe64(uid); - blk->suid = htobe64(suid); - blk->sversion = htonl(sver); - blk->schemaLen = htonl(schemaLen); - blk->numOfRows = htonl(rows); - blk->dataLen = htonl(totalLen); - subReq->length += sizeof(SSubmitBlk) + schemaLen + totalLen; - subReq->numOfBlocks++; - taosMemoryFreeClear(pTableMeta); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); + pCreateReqDst = NULL; } - pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == pQuery) { - uError("create SQuery error"); - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->haveResultSet = false; - pQuery->msgType = TDMT_VND_SUBMIT; - pQuery->pRoot = (SNode*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == pQuery->pRoot) { - uError("create pQuery->pRoot error"); - code = TSDB_CODE_OUT_OF_MEMORY; + code = smlBuildOutput(pQuery, pVgHash); + if (code != TSDB_CODE_SUCCESS) { + uError("smlBuildOutput failed"); goto end; } - SVnodeModifOpStmt* nodeStmt = (SVnodeModifOpStmt*)(pQuery->pRoot); - - int32_t numOfVg = taosHashGetSize(pVgHash); - nodeStmt->pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - - VgData* vData = (VgData*)taosHashIterate(pVgHash, NULL); - while (vData) { - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; - } - dst->vg = vData->vg; - SSubmitReq* subReq = (SSubmitReq*)(vData->data); - dst->numOfTables = subReq->numOfBlocks; - dst->size = subReq->length; - dst->pData = (char*)subReq; - vData->data = NULL; // no need free - subReq->header.vgId = htonl(dst->vg.vgId); - subReq->version = htonl(1); - subReq->header.contLen = htonl(subReq->length); - subReq->length = htonl(subReq->length); - subReq->numOfBlocks = htonl(subReq->numOfBlocks); - taosArrayPush(nodeStmt->pDataBlocks, &dst); - vData = (VgData*)taosHashIterate(pVgHash, vData); - } launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: tDeleteSTaosxRsp(&rspObj.rsp); - rspObj.resInfo.pRspMsg = NULL; - doFreeReqResultInfo(&rspObj.resInfo); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); taosHashCleanup(pVgHash); taosMemoryFreeClear(pTableMeta); - taosMemoryFree(schemaContent); + if (pCreateReqDst) { + tdDestroySVCreateTbReq(pCreateReqDst); + taosMemoryFree(pCreateReqDst); + } return code; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 0694f4c457294ad66028184e7e685d8fab7f517c..b83e7ee976cb8083e472dc176ce6c2316858fec1 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1,210 +1,101 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include #include #include #include -#include "cJSON.h" -#include "catalog.h" -#include "clientInt.h" -#include "osSemaphore.h" -#include "osThread.h" -#include "query.h" -#include "taos.h" -#include "taoserror.h" -#include "tcommon.h" -#include "tdef.h" -#include "tglobal.h" -#include "tlog.h" -#include "tmsg.h" -#include "tname.h" -#include "ttime.h" -#include "ttypes.h" - -//================================================================================================= - -#define SPACE ' ' -#define COMMA ',' -#define EQUAL '=' -#define QUOTE '"' -#define SLASH '\\' - -#define JUMP_SPACE(sql, sqlEnd) \ - while (sql < sqlEnd) { \ - if (*sql == SPACE) \ - sql++; \ - else \ - break; \ - } -// comma , -#define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) -#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) -// space -#define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) -#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) -// equal = -#define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) -#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) -// quote " -#define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) -#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) -// SLASH -#define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) - -#define IS_SLASH_LETTER(sql) \ - (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) - -#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) - -#define PROCESS_SLASH(key, keyLen) \ - for (int i = 1; i < keyLen; ++i) { \ - if (IS_SLASH_LETTER(key + i)) { \ - MOVE_FORWARD_ONE(key + i, keyLen - i); \ - i--; \ - keyLen--; \ - } \ - } - -#define IS_INVALID_COL_LEN(len) ((len) <= 0 || (len) >= TSDB_COL_NAME_LEN) -#define IS_INVALID_TABLE_LEN(len) ((len) <= 0 || (len) >= TSDB_TABLE_NAME_LEN) - -#define OTD_JSON_SUB_FIELDS_NUM 2 -#define OTD_JSON_FIELDS_NUM 4 - -#define TS "_ts" -#define TS_LEN 3 -#define VALUE "_value" -#define VALUE_LEN 6 - -#define BINARY_ADD_LEN 2 // "binary" 2 means " " -#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " - -#define MAX_RETRY_TIMES 5 -//================================================================================================= -typedef TSDB_SML_PROTOCOL_TYPE SMLProtocolType; - -typedef enum { - SCHEMA_ACTION_NULL, - SCHEMA_ACTION_CREATE_STABLE, - SCHEMA_ACTION_ADD_COLUMN, - SCHEMA_ACTION_ADD_TAG, - SCHEMA_ACTION_CHANGE_COLUMN_SIZE, - SCHEMA_ACTION_CHANGE_TAG_SIZE, -} ESchemaAction; - -typedef struct { - const char *measure; - const char *tags; - const char *cols; - const char *timestamp; - - int32_t measureLen; - int32_t measureTagsLen; - int32_t tagsLen; - int32_t colsLen; - int32_t timestampLen; -} SSmlLineInfo; - -typedef struct { - const char *sTableName; // super table name - int32_t sTableNameLen; - char childTableName[TSDB_TABLE_NAME_LEN]; - uint64_t uid; - - SArray *tags; - - // if info->formatData is true, elements are SArray. - // if info->formatData is false, elements are SHashObj for find by key quickly - SArray *cols; -} SSmlTableInfo; - -typedef struct { - SArray *tags; // save the origin order to create table - SHashObj *tagHash; // elements are - - SArray *cols; - SHashObj *colHash; - - STableMeta *tableMeta; -} SSmlSTableMeta; - -typedef struct { - int32_t len; - char *buf; -} SSmlMsgBuf; - -typedef struct { - int32_t code; - int32_t lineNum; - - int32_t numOfSTables; - int32_t numOfCTables; - int32_t numOfCreateSTables; - int32_t numOfAlterColSTables; - int32_t numOfAlterTagSTables; - - int64_t parseTime; - int64_t schemaTime; - int64_t insertBindTime; - int64_t insertRpcTime; - int64_t endTime; -} SSmlCostInfo; - -typedef struct { - int64_t id; - - SMLProtocolType protocol; - int8_t precision; - bool dataFormat; // true means that the name and order of keys in each line are the same(only for influx protocol) - bool isRawLine; - int32_t ttl; - - SHashObj *childTables; - SHashObj *superTables; - SHashObj *pVgHash; - void *exec; - - STscObj *taos; - SCatalog *pCatalog; - SRequestObj *pRequest; - SQuery *pQuery; - - SSmlCostInfo cost; - SSmlMsgBuf msgBuf; - SHashObj *dumplicateKey; // for dumplicate key - SArray *colsContainer; // for cols parse, if dataFormat == false - int32_t uid; // used for automatic create child table - - cJSON *root; // for parse json -} SSmlHandle; -//================================================================================================= - -//================================================================================================= -static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGenId() { - int64_t id; - - do { - id = atomic_add_fetch_64(&linesSmlHandleId, 1); - } while (id == 0); +#include "clientSml.h" + +int64_t smlToMilli[3] = {3600000LL, 60000LL, 1000LL}; +int64_t smlFactorNS[3] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; +int64_t smlFactorS[3] = {1000LL, 1000000LL, 1000000000LL}; + +void *nodeListGet(NodeList *list, const void *key, int32_t len, _equal_fn_sml fn) { + NodeList *tmp = list; + while (tmp) { + if (fn == NULL) { + if (tmp->data.used && tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return tmp->data.value; + } + } else { + if (tmp->data.used && fn(tmp->data.key, key) == 0) { + return tmp->data.value; + } + } - return id; + tmp = tmp->next; + } + return NULL; } -static inline bool smlDoubleToInt64OverFlow(double num) { - if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; - return false; +int nodeListSet(NodeList **list, const void *key, int32_t len, void *value, _equal_fn_sml fn) { + NodeList *tmp = *list; + while (tmp) { + if (!tmp->data.used) break; + if (fn == NULL) { + if (tmp->data.keyLen == len && memcmp(tmp->data.key, key, len) == 0) { + return -1; + } + } else { + if (tmp->data.keyLen == len && fn(tmp->data.key, key) == 0) { + return -1; + } + } + + tmp = tmp->next; + } + if (tmp) { + tmp->data.key = key; + tmp->data.keyLen = len; + tmp->data.value = value; + tmp->data.used = true; + } else { + NodeList *newNode = (NodeList *)taosMemoryCalloc(1, sizeof(NodeList)); + if (newNode == NULL) { + return -1; + } + newNode->data.key = key; + newNode->data.keyLen = len; + newNode->data.value = value; + newNode->data.used = true; + newNode->next = *list; + *list = newNode; + } + return 0; } -static inline bool smlCheckDuplicateKey(const char *key, int32_t keyLen, SHashObj *pHash) { - void *val = taosHashGet(pHash, key, keyLen); - if (val) { - return true; +int nodeListSize(NodeList *list) { + int cnt = 0; + while (list) { + if (list->data.used) + cnt++; + else + break; + list = list->next; } - taosHashPut(pHash, key, keyLen, key, 1); + return cnt; +} + +inline bool smlDoubleToInt64OverFlow(double num) { + if (num >= (double)INT64_MAX || num <= (double)INT64_MIN) return true; return false; } -static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { +int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { if (pBuf->buf) { memset(pBuf->buf, 0, pBuf->len); if (msg1) strncat(pBuf->buf, msg1, pBuf->len); @@ -217,391 +108,307 @@ static int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const return TSDB_CODE_SML_INVALID_DATA; } -static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, - ESchemaAction *action, SSmlHandle *info) { - uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; - if (index) { - if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && - (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || - (colField[*index].type == TSDB_DATA_TYPE_NCHAR && - ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { - if (isTag) { - *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; - } else { - *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; - } - } - } else { - if (isTag) { - *action = SCHEMA_ACTION_ADD_TAG; - } else { - *action = SCHEMA_ACTION_ADD_COLUMN; - } +int64_t smlGetTimeValue(const char *value, int32_t len, uint8_t fromPrecision, uint8_t toPrecision) { + char *endPtr = NULL; + int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); + if (unlikely(value + len != endPtr)) { + return -1; } - return 0; -} -static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { - int32_t result = 1; - while (result <= length) { - result *= 2; - } - if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + if (unlikely(fromPrecision >= TSDB_TIME_PRECISION_HOURS)) { + int64_t unit = smlToMilli[fromPrecision - TSDB_TIME_PRECISION_HOURS]; + if (unit > INT64_MAX / tsInt64) { + return -1; + } + tsInt64 *= unit; + fromPrecision = TSDB_TIME_PRECISION_MILLI; } - if (type == TSDB_DATA_TYPE_NCHAR) { - result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; - } else if (type == TSDB_DATA_TYPE_BINARY) { - result = result + VARSTR_HEADER_SIZE; - } - return result; + return convertTimePrecision(tsInt64, fromPrecision, toPrecision); } -static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - ESchemaAction *action, bool isTag) { - int32_t code = TSDB_CODE_SUCCESS; - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - if (j == 0 && !isTag) continue; - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); - if (code != TSDB_CODE_SUCCESS) { - return code; - } +int8_t smlGetTsTypeByLen(int32_t len) { + if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { + return TSDB_TIME_PRECISION_SECONDS; + } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { + return TSDB_TIME_PRECISION_MILLI; + } else { + return -1; } - return TSDB_CODE_SUCCESS; } -static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { - SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - int32_t i = 0; - for (; i < length; i++) { - taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); +SSmlTableInfo *smlBuildTableInfo(int numRows, const char *measure, int32_t measureLen) { + SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); + if (!tag) { + return NULL; } - if (isTag) { - i = 0; - } else { - i = 1; - } - for (; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { - taosHashCleanup(hashTmp); - return -1; - } - } - taosHashCleanup(hashTmp); - return 0; -} + tag->sTableName = measure; + tag->sTableNameLen = measureLen; -static int32_t getBytes(uint8_t type, int32_t length) { - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - return smlFindNearestPowerOf2(length, type); - } else { - return tDataTypes[type].bytes; + tag->cols = taosArrayInit(numRows, POINTER_BYTES); + if (tag->cols == NULL) { + uError("SML:smlBuildTableInfo failed to allocate memory"); + goto cleanup; } + + // tag->tags = taosArrayInit(16, sizeof(SSmlKv)); + // if (tag->tags == NULL) { + // uError("SML:smlBuildTableInfo failed to allocate memory"); + // goto cleanup; + // } + return tag; + +cleanup: + taosMemoryFree(tag); + return NULL; } -static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, - SArray *results, int32_t numOfCols, bool isTag) { - for (int j = 0; j < taosArrayGetSize(cols); ++j) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, j); - ESchemaAction action = SCHEMA_ACTION_NULL; - smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); - if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { - SField field = {0}; - field.type = kv->type; - field.bytes = getBytes(kv->type, kv->length); - memcpy(field.name, kv->key, kv->keyLen); - taosArrayPush(results, &field); - } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); - uint16_t newIndex = *index; - if (isTag) newIndex -= numOfCols; - SField *field = (SField *)taosArrayGet(results, newIndex); - field->bytes = getBytes(kv->type, kv->length); +static int32_t smlParseTableName(SArray *tags, char *childTableName) { + size_t childTableNameLen = strlen(tsSmlChildTableName); + if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; + + for (int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); + // handle child table name + if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { + memset(childTableName, 0, TSDB_TABLE_NAME_LEN); + strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + break; } } + return TSDB_CODE_SUCCESS; } -// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, -// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ -static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, - ESchemaAction action) { - SRequestObj *pRequest = NULL; - SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SCmdMsgInfo pCmdMsg = {0}; +int32_t smlSetCTableName(SSmlTableInfo *oneTable) { + smlParseTableName(oneTable->tags, oneTable->childTableName); - // put front for free - pReq.numOfColumns = taosArrayGetSize(pColumns); - pReq.pColumns = pColumns; - pReq.numOfTags = taosArrayGetSize(pTags); - pReq.pTags = pTags; + if (strlen(oneTable->childTableName) == 0) { + SArray *dst = taosArrayDup(oneTable->tags, NULL); + RandTableName rName = {dst, oneTable->sTableName, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; + buildChildTableName(&rName); + taosArrayDestroy(dst); } + return TSDB_CODE_SUCCESS; +} - pRequest->syncQuery = true; - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; +SSmlSTableMeta *smlBuildSTableMeta(bool isDataFormat) { + SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); + if (!meta) { + return NULL; } - if (action == SCHEMA_ACTION_CREATE_STABLE) { - pReq.colVer = 1; - pReq.tagVer = 1; - pReq.suid = 0; - pReq.source = TD_REQ_FROM_APP; - } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { - pReq.colVer = pTableMeta->sversion; - pReq.tagVer = pTableMeta->tversion + 1; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { - pReq.colVer = pTableMeta->sversion + 1; - pReq.tagVer = pTableMeta->tversion; - pReq.suid = pTableMeta->uid; - pReq.source = TD_REQ_FROM_TAOX; - } + if (unlikely(!isDataFormat)) { + meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->tagHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } - if (pReq.numOfTags == 0) { - pReq.numOfTags = 1; - SField field = {0}; - field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; - strcpy(field.name, tsSmlTagName); - taosArrayPush(pReq.pTags, &field); + meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (meta->colHash == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; + } } - pReq.commentLen = -1; - pReq.igExists = true; - tNameExtractFullName(pName, pReq.name); - - pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - pCmdMsg.msgType = TDMT_MND_CREATE_STB; - pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); - pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); - if (NULL == pCmdMsg.pMsg) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto end; + meta->tags = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->tags == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - - SQuery pQuery; - memset(&pQuery, 0, sizeof(pQuery)); - pQuery.execMode = QUERY_EXEC_MODE_RPC; - pQuery.pCmdMsg = &pCmdMsg; - pQuery.msgType = pQuery.pCmdMsg->msgType; - pQuery.stableQuery = true; - - launchQueryImpl(pRequest, &pQuery, true, NULL); - if (pRequest->code == TSDB_CODE_SUCCESS) { - catalogRemoveTableMeta(info->pCatalog, pName); + meta->cols = taosArrayInit(32, sizeof(SSmlKv)); + if (meta->cols == NULL) { + uError("SML:smlBuildSTableMeta failed to allocate memory"); + goto cleanup; } - code = pRequest->code; - taosMemoryFree(pCmdMsg.pMsg); + return meta; -end: - destroyRequest(pRequest); - tFreeSMCreateStbReq(&pReq); - return code; +cleanup: + taosMemoryFree(meta); + return NULL; } -static int32_t smlModifyDBSchemas(SSmlHandle *info) { - int32_t code = 0; - SHashObj *hashTmp = NULL; - STableMeta *pTableMeta = NULL; +// uint16_t smlCalTypeSum(char* endptr, int32_t left){ +// uint16_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } - SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; - tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); +#define RETURN_FALSE \ + smlBuildInvalidDataMsg(msg, "invalid data", pVal); \ + return false; - SRequestConnInfo conn = {0}; - conn.pTrans = info->taos->pAppInfo->pTransporter; - conn.requestId = info->pRequest->requestId; - conn.requestObjRefId = info->pRequest->self; - conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - - SSmlSTableMeta **tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); - while (tableMetaSml) { - SSmlSTableMeta *sTableData = *tableMetaSml; - bool needCheckMeta = false; // for multi thread - - size_t superTableLen = 0; - void *superTable = taosHashGetKey(tableMetaSml, &superTableLen); - memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); - memcpy(pName.tname, superTable, superTableLen); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { - SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); - SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); - smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); - smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - info->cost.numOfCreateSTables++; - taosMemoryFreeClear(pTableMeta); - - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } - } else if (code == TSDB_CODE_SUCCESS) { - hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, - HASH_NO_LOCK); - for (uint16_t i = pTableMeta->tableInfo.numOfColumns; - i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); - } - - ESchemaAction action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, - pTableMeta->tableInfo.numOfColumns, true); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterTagSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - } +#define SET_DOUBLE \ + kvVal->type = TSDB_DATA_TYPE_DOUBLE; \ + kvVal->d = result; + +#define SET_FLOAT \ + if (!IS_VALID_FLOAT(result)) { \ + smlBuildInvalidDataMsg(msg, "float out of range[-3.402823466e+38,3.402823466e+38]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_FLOAT; \ + kvVal->f = (float)result; + +#define SET_BIGINT \ + if (smlDoubleToInt64OverFlow(result)) { \ + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = (int64_t)result; + +#define SET_INT \ + if (!IS_VALID_INT(result)) { \ + smlBuildInvalidDataMsg(msg, "int out of range[-2147483648,2147483647]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_INT; \ + kvVal->i = result; + +#define SET_SMALL_INT \ + if (!IS_VALID_SMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "small int our of range[-32768,32767]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_SMALLINT; \ + kvVal->i = result; + +#define SET_UBIGINT \ + if (result >= (double)UINT64_MAX || result < 0) { \ + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; \ + return true; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = result; + +#define SET_UINT \ + if (!IS_VALID_UINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned int out of range[0,4294967295]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UINT; \ + kvVal->u = result; + +#define SET_USMALL_INT \ + if (!IS_VALID_USMALLINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned small int out of rang[0,65535]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_USMALLINT; \ + kvVal->u = result; + +#define SET_TINYINT \ + if (!IS_VALID_TINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "tiny int out of range[-128,127]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_TINYINT; \ + kvVal->i = result; + +#define SET_UTINYINT \ + if (!IS_VALID_UTINYINT(result)) { \ + smlBuildInvalidDataMsg(msg, "unsigned tiny int out of range[0,255]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UTINYINT; \ + kvVal->u = result; + +bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { + const char *pVal = kvVal->value; + int32_t len = kvVal->length; + char *endptr = NULL; + double result = taosStr2Double(pVal, &endptr); + if (pVal == endptr) { + RETURN_FALSE + } - taosHashClear(hashTmp); - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { - taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); + int32_t left = len - (endptr - pVal); + if (left == 0) { + SET_DOUBLE + } else if (left == 3) { + if (endptr[0] == 'f' || endptr[0] == 'F') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_DOUBLE + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_FLOAT + } else { + RETURN_FALSE } - action = SCHEMA_ACTION_NULL; - code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); - if (code != TSDB_CODE_SUCCESS) { - goto end; + } else if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_BIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_INT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_SMALL_INT + } else { + RETURN_FALSE } - if (action != SCHEMA_ACTION_NULL) { - SArray *pColumns = - taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); - SArray *pTags = - taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - - for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { - SField field = {0}; - field.type = pTableMeta->schema[i].type; - field.bytes = pTableMeta->schema[i].bytes; - strcpy(field.name, pTableMeta->schema[i].name); - if (i < pTableMeta->tableInfo.numOfColumns) { - taosArrayPush(pColumns, &field); - } else { - taosArrayPush(pTags, &field); - } - } - - smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, - pTableMeta->tableInfo.numOfColumns, false); - - code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); - goto end; - } - - info->cost.numOfAlterColSTables++; - taosMemoryFreeClear(pTableMeta); - code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); - goto end; - } + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '6' && endptr[2] == '4') { + SET_UBIGINT + } else if (endptr[1] == '3' && endptr[2] == '2') { + SET_UINT + } else if (endptr[1] == '1' && endptr[2] == '6') { + SET_USMALL_INT + } else { + RETURN_FALSE } - - needCheckMeta = true; - taosHashCleanup(hashTmp); - hashTmp = NULL; } else { - uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); - goto end; + RETURN_FALSE } - - if (needCheckMeta) { - code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, - sTableData->tags, true); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); - goto end; + } else if (left == 2) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + if (endptr[1] == '8') { + SET_TINYINT + } else { + RETURN_FALSE } - code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); - goto end; + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + if (endptr[1] == '8') { + SET_UTINYINT + } else { + RETURN_FALSE } + } else { + RETURN_FALSE } - - sTableData->tableMeta = pTableMeta; - - tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); + } else if (left == 1) { + if (endptr[0] == 'i' || endptr[0] == 'I') { + SET_BIGINT + } else if (endptr[0] == 'u' || endptr[0] == 'U') { + SET_UBIGINT + } else { + RETURN_FALSE + } + } else { + RETURN_FALSE; } - return 0; - -end: - taosHashCleanup(hashTmp); - taosMemoryFreeClear(pTableMeta); -// catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); - return code; + return true; } -static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { +bool smlParseNumberOld(SSmlKv *kvVal, SSmlMsgBuf *msg) { const char *pVal = kvVal->value; int32_t len = kvVal->length; char *endptr = NULL; @@ -699,764 +506,462 @@ static bool smlParseNumber(SSmlKv *kvVal, SSmlMsgBuf *msg) { return true; } -static bool smlParseBool(SSmlKv *kvVal) { - const char *pVal = kvVal->value; - int32_t len = kvVal->length; - if ((len == 1) && (pVal[0] == 't' || pVal[0] == 'T')) { - kvVal->i = true; - return true; - } +STableMeta *smlGetMeta(SSmlHandle *info, const void *measure, int32_t measureLen) { + STableMeta *pTableMeta = NULL; - if ((len == 1) && (pVal[0] == 'f' || pVal[0] == 'F')) { - kvVal->i = false; - return true; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - if ((len == 4) && !strncasecmp(pVal, "true", len)) { - kvVal->i = true; - return true; - } - if ((len == 5) && !strncasecmp(pVal, "false", len)) { - kvVal->i = false; - return true; - } - return false; -} + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, measure, measureLen); -static bool smlIsBinary(const char *pVal, uint16_t len) { - // binary: "abc" - if (len < 2) { - return false; - } - if (pVal[0] == '"' && pVal[len - 1] == '"') { - return true; + int32_t code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + return NULL; } - return false; + return pTableMeta; } -static bool smlIsNchar(const char *pVal, uint16_t len) { - // nchar: L"abc" - if (len < 3) { - return false; - } - if ((pVal[0] == 'l' || pVal[0] == 'L') && pVal[1] == '"' && pVal[len - 1] == '"') { - return true; - } - return false; -} +static int64_t smlGenId() { + static volatile int64_t linesSmlHandleId = 0; -static int64_t smlGetTimeValue(const char *value, int32_t len, int8_t type) { - char *endPtr = NULL; - int64_t tsInt64 = taosStr2Int64(value, &endPtr, 10); - if (value + len != endPtr) { - return -1; - } - double ts = tsInt64; - switch (type) { - case TSDB_TIME_PRECISION_HOURS: - ts *= NANOSECOND_PER_HOUR; - tsInt64 *= NANOSECOND_PER_HOUR; - break; - case TSDB_TIME_PRECISION_MINUTES: - ts *= NANOSECOND_PER_MINUTE; - tsInt64 *= NANOSECOND_PER_MINUTE; - break; - case TSDB_TIME_PRECISION_SECONDS: - ts *= NANOSECOND_PER_SEC; - tsInt64 *= NANOSECOND_PER_SEC; - break; - case TSDB_TIME_PRECISION_MILLI: - ts *= NANOSECOND_PER_MSEC; - tsInt64 *= NANOSECOND_PER_MSEC; - break; - case TSDB_TIME_PRECISION_MICRO: - ts *= NANOSECOND_PER_USEC; - tsInt64 *= NANOSECOND_PER_USEC; - break; - case TSDB_TIME_PRECISION_NANO: - break; - default: - ASSERT(0); - } - if (ts >= (double)INT64_MAX || ts < 0) { - return -1; - } + int64_t id = 0; + do { + id = atomic_add_fetch_64(&linesSmlHandleId, 1); + } while (id == 0); - return tsInt64; + return id; } -static int8_t smlGetTsTypeByLen(int32_t len) { - if (len == TSDB_TIME_PRECISION_SEC_DIGITS) { - return TSDB_TIME_PRECISION_SECONDS; - } else if (len == TSDB_TIME_PRECISION_MILLI_DIGITS) { - return TSDB_TIME_PRECISION_MILLI; +static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSmlKv *kv, bool isTag, + ESchemaAction *action, SSmlHandle *info) { + uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; + if (index) { + if (colField[*index].type != kv->type) { + uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, + kv->key, colField[*index].type, kv->type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if ((colField[*index].type == TSDB_DATA_TYPE_VARCHAR && + (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || + (colField[*index].type == TSDB_DATA_TYPE_NCHAR && + ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { + if (isTag) { + *action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + *action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + } } else { - return -1; + if (isTag) { + *action = SCHEMA_ACTION_ADD_TAG; + } else { + *action = SCHEMA_ACTION_ADD_COLUMN; + } } + return 0; } -static int8_t smlGetTsTypeByPrecision(int8_t precision) { - switch (precision) { - case TSDB_SML_TIMESTAMP_HOURS: - return TSDB_TIME_PRECISION_HOURS; - case TSDB_SML_TIMESTAMP_MILLI_SECONDS: - return TSDB_TIME_PRECISION_MILLI; - case TSDB_SML_TIMESTAMP_NANO_SECONDS: - case TSDB_SML_TIMESTAMP_NOT_CONFIGURED: - return TSDB_TIME_PRECISION_NANO; - case TSDB_SML_TIMESTAMP_MICRO_SECONDS: - return TSDB_TIME_PRECISION_MICRO; - case TSDB_SML_TIMESTAMP_SECONDS: - return TSDB_TIME_PRECISION_SECONDS; - case TSDB_SML_TIMESTAMP_MINUTES: - return TSDB_TIME_PRECISION_MINUTES; - default: - return -1; +static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { + int32_t result = 1; + while (result <= length) { + result *= 2; } -} - -static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { - if (len == 0 || (len == 1 && data[0] == '0')) { - return taosGetTimestampNs(); + if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + result = (TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; } - int8_t tsType = smlGetTsTypeByPrecision(info->precision); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp precision", NULL); - return -1; + if (type == TSDB_DATA_TYPE_NCHAR) { + result = result * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else if (type == TSDB_DATA_TYPE_BINARY) { + result = result + VARSTR_HEADER_SIZE; } + return result; +} - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; +static int32_t smlProcessSchemaAction(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + ESchemaAction *action, bool isTag) { + int32_t code = TSDB_CODE_SUCCESS; + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + if (j == 0 && !isTag) continue; + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + code = smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, action, info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return ts; + return TSDB_CODE_SUCCESS; } -static int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { - if (!data) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); - return -1; - } - if (len == 1 && data[0] == '0') { - return taosGetTimestampNs(); - } - int8_t tsType = smlGetTsTypeByLen(len); - if (tsType == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, - "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); - return -1; - } - int64_t ts = smlGetTimeValue(data, len, tsType); - if (ts == -1) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); - return -1; +static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool isTag) { + SHashObj *hashTmp = taosHashInit(length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t i = 0; + for (; i < length; i++) { + taosHashPut(hashTmp, schema[i].name, strlen(schema[i].name), &i, SHORT_BYTES); } - return ts; -} -static int32_t smlParseTS(SSmlHandle *info, const char *data, int32_t len, SArray *cols) { - int64_t ts = 0; - if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - // uError("SML:data:%s,len:%d", data, len); - ts = smlParseInfluxTime(info, data, len); - } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ts = smlParseOpenTsdbTime(info, data, len); + if (isTag) { + i = 0; } else { - ASSERT(0); + i = 1; } - uDebug("SML:0x%" PRIx64 " smlParseTS:%" PRId64, info->id, ts); - - if (ts <= 0) { - uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); - return TSDB_CODE_INVALID_TIMESTAMP; + for (; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + if (taosHashGet(hashTmp, kv->key, kv->keyLen) == NULL) { + taosHashCleanup(hashTmp); + return -1; + } } + taosHashCleanup(hashTmp); + return 0; +} - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; +static int32_t getBytes(uint8_t type, int32_t length) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + return smlFindNearestPowerOf2(length, type); + } else { + return tDataTypes[type].bytes; } - - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = ts; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - - return TSDB_CODE_SUCCESS; } -static int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { - // binary - if (smlIsBinary(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_BINARY; - pVal->length -= BINARY_ADD_LEN; - if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - pVal->value += (BINARY_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - // nchar - if (smlIsNchar(pVal->value, pVal->length)) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - pVal->length -= NCHAR_ADD_LEN; - if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; +static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashObj *schemaHash, SArray *cols, + SArray *results, int32_t numOfCols, bool isTag) { + for (int j = 0; j < taosArrayGetSize(cols); ++j) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, j); + ESchemaAction action = SCHEMA_ACTION_NULL; + smlGenerateSchemaAction(schemaField, schemaHash, kv, isTag, &action, info); + if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_ADD_TAG) { + SField field = {0}; + field.type = kv->type; + field.bytes = getBytes(kv->type, kv->length); + memcpy(field.name, kv->key, kv->keyLen); + taosArrayPush(results, &field); + } else if (action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + uint16_t *index = (uint16_t *)taosHashGet(schemaHash, kv->key, kv->keyLen); + uint16_t newIndex = *index; + if (isTag) newIndex -= numOfCols; + SField *field = (SField *)taosArrayGet(results, newIndex); + field->bytes = getBytes(kv->type, kv->length); } - pVal->value += (NCHAR_ADD_LEN - 1); - return TSDB_CODE_SUCCESS; - } - - // bool - if (smlParseBool(pVal)) { - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; - } - // number - if (smlParseNumber(pVal, msg)) { - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_TSC_INVALID_VALUE; + return TSDB_CODE_SUCCESS; } -static int32_t smlParseInfluxString(const char *sql, const char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; - JUMP_SPACE(sql, sqlEnd) - if (*sql == COMMA) return TSDB_CODE_SML_INVALID_DATA; - elements->measure = sql; - - // parse measure - while (sql < sqlEnd) { - if ((sql != elements->measure) && IS_SLASH_LETTER(sql)) { - MOVE_FORWARD_ONE(sql, sqlEnd - sql); - sqlEnd--; - continue; - } - if (IS_COMMA(sql)) { - break; - } +// static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SSmlSTableMeta *sTableData, +// int32_t colVer, int32_t tagVer, int8_t source, uint64_t suid){ +static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SArray *pTags, STableMeta *pTableMeta, + ESchemaAction action) { + SRequestObj *pRequest = NULL; + SMCreateStbReq pReq = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SCmdMsgInfo pCmdMsg = {0}; - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->measureLen = sql - elements->measure; - if (IS_INVALID_TABLE_LEN(elements->measureLen)) { - smlBuildInvalidDataMsg(msg, "measure is empty or too large than 192", NULL); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } + // put front for free + pReq.numOfColumns = taosArrayGetSize(pColumns); + pReq.pColumns = pColumns; + pReq.numOfTags = taosArrayGetSize(pTags); + pReq.pTags = pTags; - // parse tag - if (*sql == SPACE) { - elements->tagsLen = 0; - } else { - if (*sql == COMMA) sql++; - elements->tags = sql; - while (sql < sqlEnd) { - if (IS_SPACE(sql)) { - break; - } - sql++; - } - elements->tagsLen = sql - elements->tags; + code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - elements->measureTagsLen = sql - elements->measure; - // parse cols - JUMP_SPACE(sql, sqlEnd) - elements->cols = sql; - bool isInQuote = false; - while (sql < sqlEnd) { - if (IS_QUOTE(sql)) { - isInQuote = !isInQuote; - } - if (!isInQuote && IS_SPACE(sql)) { - break; - } - sql++; - } - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", elements->cols); - return TSDB_CODE_SML_INVALID_DATA; - } - elements->colsLen = sql - elements->cols; - if (elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; + pRequest->syncQuery = true; + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; } - // parse timestamp - JUMP_SPACE(sql, sqlEnd) - elements->timestamp = sql; - while (sql < sqlEnd) { - if (isspace(*sql)) { - break; - } - sql++; + if (action == SCHEMA_ACTION_CREATE_STABLE) { + pReq.colVer = 1; + pReq.tagVer = 1; + pReq.suid = 0; + pReq.source = TD_REQ_FROM_APP; + } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { + pReq.colVer = pTableMeta->sversion; + pReq.tagVer = pTableMeta->tversion + 1; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; + } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { + pReq.colVer = pTableMeta->sversion + 1; + pReq.tagVer = pTableMeta->tversion; + pReq.suid = pTableMeta->uid; + pReq.source = TD_REQ_FROM_TAOX; } - elements->timestampLen = sql - elements->timestamp; - - return TSDB_CODE_SUCCESS; -} -static void smlParseTelnetElement(const char **sql, const char *sqlEnd, const char **data, int32_t *len) { - while (*sql < sqlEnd) { - if (**sql != SPACE && !(*data)) { - *data = *sql; - } else if (**sql == SPACE && *data) { - *len = *sql - *data; - break; - } - (*sql)++; + if (pReq.numOfTags == 0) { + pReq.numOfTags = 1; + SField field = {0}; + field.type = TSDB_DATA_TYPE_NCHAR; + field.bytes = 1; + strcpy(field.name, tsSmlTagName); + taosArrayPush(pReq.pTags, &field); } -} - -static int32_t smlParseTelnetTags(const char *data, const char *sqlEnd, SArray *cols, char *childTableName, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - const char *sql = data; - size_t childTableNameLen = strlen(tsSmlChildTableName); - while (sql < sqlEnd) { - JUMP_SPACE(sql, sqlEnd) - if (*sql == '\0') break; - - const char *key = sql; - int32_t keyLen = 0; - - // parse key - while (sql < sqlEnd) { - if (*sql == SPACE) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (*sql == EQUAL) { - keyLen = sql - key; - sql++; - break; - } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } - - // parse value - const char *value = sql; - int32_t valueLen = 0; - while (sql < sqlEnd) { - // parse value - if (*sql == SPACE) { - break; - } - if (*sql == EQUAL) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - sql++; - } - valueLen = sql - value; - - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - // handle child table name - if (childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - kv->type = TSDB_DATA_TYPE_NCHAR; + pReq.commentLen = -1; + pReq.igExists = true; + tNameExtractFullName(pName, pReq.name); - taosArrayPush(cols, &kv); + pCmdMsg.epSet = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + pCmdMsg.msgType = TDMT_MND_CREATE_STB; + pCmdMsg.msgLen = tSerializeSMCreateStbReq(NULL, 0, &pReq); + pCmdMsg.pMsg = taosMemoryMalloc(pCmdMsg.msgLen); + if (NULL == pCmdMsg.pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; } + tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - return TSDB_CODE_SUCCESS; -} + SQuery pQuery; + memset(&pQuery, 0, sizeof(pQuery)); + pQuery.execMode = QUERY_EXEC_MODE_RPC; + pQuery.pCmdMsg = &pCmdMsg; + pQuery.msgType = pQuery.pCmdMsg->msgType; + pQuery.stableQuery = true; -// format: =[ =] -static int32_t smlParseTelnetString(SSmlHandle *info, const char *sql, const char *sqlEnd, SSmlTableInfo *tinfo, - SArray *cols) { - if (!sql) return TSDB_CODE_SML_INVALID_DATA; + launchQueryImpl(pRequest, &pQuery, true, NULL); - // parse metric - smlParseTelnetElement(&sql, sqlEnd, &tinfo->sTableName, &tinfo->sTableNameLen); - if (!(tinfo->sTableName) || IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + if (pRequest->code == TSDB_CODE_SUCCESS) { + catalogRemoveTableMeta(info->pCatalog, pName); } + code = pRequest->code; + taosMemoryFree(pCmdMsg.pMsg); - // parse timestamp - const char *timestamp = NULL; - int32_t tLen = 0; - smlParseTelnetElement(&sql, sqlEnd, ×tamp, &tLen); - if (!timestamp || tLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return TSDB_CODE_SML_INVALID_DATA; - } +end: + destroyRequest(pRequest); + tFreeSMCreateStbReq(&pReq); + return code; +} - int32_t ret = smlParseTS(info, timestamp, tLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); - return ret; +static int32_t smlModifyDBSchemas(SSmlHandle *info) { + if (info->dataFormat && !info->needModifySchema) { + return TSDB_CODE_SUCCESS; } + int32_t code = 0; + SHashObj *hashTmp = NULL; + STableMeta *pTableMeta = NULL; - // parse value - const char *value = NULL; - int32_t valueLen = 0; - smlParseTelnetElement(&sql, sqlEnd, &value, &valueLen); - if (!value || valueLen == 0) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); - return TSDB_CODE_TSC_INVALID_VALUE; - } + SName pName = {TSDB_TABLE_NAME_T, info->taos->acctId, {0}, {0}}; + tstrncpy(pName.dbname, info->pRequest->pDb, sizeof(pName.dbname)); - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - kv->value = value; - kv->length = valueLen; - if ((ret = smlParseValue(kv, &info->msgBuf)) != TSDB_CODE_SUCCESS) { - return ret; - } + SRequestConnInfo conn = {0}; + conn.pTrans = info->taos->pAppInfo->pTransporter; + conn.requestId = info->pRequest->requestId; + conn.requestObjRefId = info->pRequest->self; + conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); - // parse tags - ret = smlParseTelnetTags(sql, sqlEnd, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); - return ret; - } + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); + while (tmp) { + SSmlSTableMeta *sTableData = *tmp; + bool needCheckMeta = false; // for multi thread - return TSDB_CODE_SUCCESS; -} + size_t superTableLen = 0; + void *superTable = taosHashGetKey(tmp, &superTableLen); + memset(pName.tname, 0, TSDB_TABLE_NAME_LEN); + memcpy(pName.tname, superTable, superTableLen); -static int32_t smlParseCols(const char *data, int32_t len, SArray *cols, char *childTableName, bool isTag, - SHashObj *dumplicateKey, SSmlMsgBuf *msg) { - if (len == 0) { - return TSDB_CODE_SUCCESS; - } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); - size_t childTableNameLen = strlen(tsSmlChildTableName); - const char *sql = data; - while (sql < data + len) { - const char *key = sql; - int32_t keyLen = 0; + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { + SArray *pColumns = taosArrayInit(taosArrayGetSize(sTableData->cols), sizeof(SField)); + SArray *pTags = taosArrayInit(taosArrayGetSize(sTableData->tags), sizeof(SField)); + smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); + smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); - while (sql < data + len) { - // parse key - if (IS_COMMA(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; - } - if (IS_EQUAL(sql)) { - keyLen = sql - key; - sql++; - break; + code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; } - sql++; - } - - if (IS_INVALID_COL_LEN(keyLen)) { - smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { - smlBuildInvalidDataMsg(msg, "dumplicate key", key); - return TSDB_CODE_TSC_DUP_NAMES; - } + info->cost.numOfCreateSTables++; + taosMemoryFreeClear(pTableMeta); - // parse value - const char *value = sql; - int32_t valueLen = 0; - bool isInQuote = false; - while (sql < data + len) { - // parse value - if (!isTag && IS_QUOTE(sql)) { - isInQuote = !isInQuote; - sql++; - continue; + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; } - if (!isInQuote && IS_COMMA(sql)) { - break; + } else if (code == TSDB_CODE_SUCCESS) { + hashTmp = taosHashInit(pTableMeta->tableInfo.numOfTags, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, + HASH_NO_LOCK); + for (uint16_t i = pTableMeta->tableInfo.numOfColumns; + i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } - if (!isInQuote && IS_EQUAL(sql)) { - smlBuildInvalidDataMsg(msg, "invalid data", sql); - return TSDB_CODE_SML_INVALID_DATA; + + ESchemaAction action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->tags, &action, true); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - sql++; - } - valueLen = sql - value; - sql++; + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - if (isInQuote) { - smlBuildInvalidDataMsg(msg, "only one quote", value); - return TSDB_CODE_SML_INVALID_DATA; - } - if (valueLen == 0) { - smlBuildInvalidDataMsg(msg, "invalid value", value); - return TSDB_CODE_SML_INVALID_DATA; - } - PROCESS_SLASH(key, keyLen) - PROCESS_SLASH(value, valueLen) + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); + } + } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->tags, pTags, + pTableMeta->tableInfo.numOfColumns, true); - // handle child table name - if (childTableName && childTableNameLen != 0 && strncmp(key, tsSmlChildTableName, keyLen) == 0) { - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - strncpy(childTableName, value, (valueLen < TSDB_TABLE_NAME_LEN ? valueLen : TSDB_TABLE_NAME_LEN)); - continue; - } + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - if (cols) taosArrayPush(cols, &kv); + info->cost.numOfAlterTagSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + } - kv->key = key; - kv->keyLen = keyLen; - kv->value = value; - kv->length = valueLen; - if (isTag) { - if (valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + taosHashClear(hashTmp); + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; i++) { + taosHashPut(hashTmp, pTableMeta->schema[i].name, strlen(pTableMeta->schema[i].name), &i, SHORT_BYTES); } - kv->type = TSDB_DATA_TYPE_NCHAR; - } else { - int32_t ret = smlParseValue(kv, msg); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + action = SCHEMA_ACTION_NULL; + code = smlProcessSchemaAction(info, pTableMeta->schema, hashTmp, sTableData->cols, &action, false); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - } - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, SSmlMsgBuf *msg) { - for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); + if (action != SCHEMA_ACTION_NULL) { + SArray *pColumns = + taosArrayInit(taosArrayGetSize(sTableData->cols) + pTableMeta->tableInfo.numOfColumns, sizeof(SField)); + SArray *pTags = + taosArrayInit(taosArrayGetSize(sTableData->tags) + pTableMeta->tableInfo.numOfTags, sizeof(SField)); - int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); - if (index) { - SSmlKv **value = (SSmlKv **)taosArrayGet(metaArray, *index); - if (kv->type != (*value)->type) { - smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); - return TSDB_CODE_SML_NOT_SAME_TYPE; - } else { - if (IS_VAR_DATA_TYPE(kv->type)) { // update string len, if bigger - if (kv->length > (*value)->length) { - *value = kv; + for (uint16_t i = 0; i < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; i++) { + SField field = {0}; + field.type = pTableMeta->schema[i].type; + field.bytes = pTableMeta->schema[i].bytes; + strcpy(field.name, pTableMeta->schema[i].name); + if (i < pTableMeta->tableInfo.numOfColumns) { + taosArrayPush(pColumns, &field); + } else { + taosArrayPush(pTags, &field); } } - } - } else { - size_t tmp = taosArrayGetSize(metaArray); - ASSERT(tmp <= INT16_MAX); - int16_t size = tmp; - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); - } - } - return TSDB_CODE_SUCCESS; -} - -static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { - for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosArrayPush(metaArray, &kv); - taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); - } -} - -static SSmlTableInfo *smlBuildTableInfo() { - SSmlTableInfo *tag = (SSmlTableInfo *)taosMemoryCalloc(sizeof(SSmlTableInfo), 1); - if (!tag) { - return NULL; - } + smlBuildFieldsList(info, pTableMeta->schema, hashTmp, sTableData->cols, pColumns, + pTableMeta->tableInfo.numOfColumns, false); - tag->cols = taosArrayInit(16, POINTER_BYTES); - if (tag->cols == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } + code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); + goto end; + } - tag->tags = taosArrayInit(16, POINTER_BYTES); - if (tag->tags == NULL) { - uError("SML:smlBuildTableInfo failed to allocate memory"); - goto cleanup; - } - return tag; + info->cost.numOfAlterColSTables++; + taosMemoryFreeClear(pTableMeta); + code = catalogRefreshTableMeta(info->pCatalog, &conn, &pName, -1); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); + goto end; + } + } -cleanup: - taosMemoryFree(tag); - return NULL; -} + needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; + } else { + uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); + goto end; + } -static void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { - if (info->dataFormat) { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SArray *kvArray = (SArray *)taosArrayGetP(tag->cols, i); - for (int j = 0; j < taosArrayGetSize(kvArray); ++j) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(kvArray, j); - taosMemoryFree(p); + if (needCheckMeta) { + code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, + sTableData->tags, true); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); + goto end; } - taosArrayDestroy(kvArray); - } - } else { - for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { - SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); - void **p1 = (void **)taosHashIterate(kvHash, NULL); - while (p1) { - taosMemoryFree(*p1); - p1 = (void **)taosHashIterate(kvHash, p1); + code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); + goto end; } - taosHashCleanup(kvHash); } - } - for (size_t i = 0; i < taosArrayGetSize(tag->tags); i++) { - SSmlKv *p = (SSmlKv *)taosArrayGetP(tag->tags, i); - taosMemoryFree(p); - } - taosArrayDestroy(tag->cols); - taosArrayDestroy(tag->tags); - taosMemoryFree(tag); -} -static int32_t smlKvTimeArrayCompare(const void *key1, const void *key2) { - SArray *s1 = *(SArray **)key1; - SArray *s2 = *(SArray **)key2; - SSmlKv *kv1 = (SSmlKv *)taosArrayGetP(s1, 0); - SSmlKv *kv2 = (SSmlKv *)taosArrayGetP(s2, 0); - ASSERT(kv1->type == TSDB_DATA_TYPE_TIMESTAMP); - ASSERT(kv2->type == TSDB_DATA_TYPE_TIMESTAMP); - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; - } -} + sTableData->tableMeta = pTableMeta; -static int32_t smlKvTimeHashCompare(const void *key1, const void *key2) { - SHashObj *s1 = *(SHashObj **)key1; - SHashObj *s2 = *(SHashObj **)key2; - SSmlKv **kv1pp = (SSmlKv **)taosHashGet(s1, TS, TS_LEN); - SSmlKv **kv2pp = (SSmlKv **)taosHashGet(s2, TS, TS_LEN); - if (!kv1pp || !kv2pp) { - uError("smlKvTimeHashCompare kv is null"); - return -1; - } - SSmlKv *kv1 = *kv1pp; - SSmlKv *kv2 = *kv2pp; - if (!kv1 || kv1->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv1"); - return -1; - } - if (!kv2 || kv2->type != TSDB_DATA_TYPE_TIMESTAMP) { - uError("smlKvTimeHashCompare kv2"); - return -1; - } - if (kv1->i < kv2->i) { - return -1; - } else if (kv1->i > kv2->i) { - return 1; - } else { - return 0; + tmp = (SSmlSTableMeta **)taosHashIterate(info->superTables, tmp); } + return 0; + +end: + taosHashCleanup(hashTmp); + taosMemoryFreeClear(pTableMeta); + // catalogRefreshTableMeta(info->pCatalog, &conn, &pName, 1); + return code; } -static int32_t smlDealCols(SSmlTableInfo *oneTable, bool dataFormat, SArray *cols) { - if (dataFormat) { - void *p = taosArraySearch(oneTable->cols, &cols, smlKvTimeArrayCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &cols); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &cols); +/* +static int32_t smlCheckDupUnit(SHashObj *dumplicateKey, SArray *tags, SSmlMsgBuf *msg){ + for(int i = 0; i < taosArrayGetSize(tags); i++) { + SSmlKv *tag = taosArrayGet(tags, i); + if (smlCheckDuplicateKey(tag->key, tag->keyLen, dumplicateKey)) { + smlBuildInvalidDataMsg(msg, "dumplicate key", tag->key); + return TSDB_CODE_TSC_DUP_NAMES; } - return TSDB_CODE_SUCCESS; - } - - SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!kvHash) { - uError("SML:smlDealCols failed to allocate memory"); - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < taosArrayGetSize(cols); i++) { - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, i); - taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); - } - - void *p = taosArraySearch(oneTable->cols, &kvHash, smlKvTimeHashCompare, TD_GT); - if (p == NULL) { - taosArrayPush(oneTable->cols, &kvHash); - } else { - taosArrayInsert(oneTable->cols, TARRAY_ELEM_IDX(oneTable->cols, p), &kvHash); } return TSDB_CODE_SUCCESS; } -static SSmlSTableMeta *smlBuildSTableMeta() { - SSmlSTableMeta *meta = (SSmlSTableMeta *)taosMemoryCalloc(sizeof(SSmlSTableMeta), 1); - if (!meta) { - return NULL; - } - meta->tagHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->tagHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static int32_t smlJudgeDupColName(SArray *cols, SArray *tags, SSmlMsgBuf *msg) { + SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int ret = smlCheckDupUnit(dumplicateKey, cols, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - - meta->colHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (meta->colHash == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; + ret = smlCheckDupUnit(dumplicateKey, tags, msg); + if(ret != TSDB_CODE_SUCCESS){ + goto end; } - meta->tags = taosArrayInit(32, POINTER_BYTES); - if (meta->tags == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; - } + end: + taosHashCleanup(dumplicateKey); + return ret; +} +*/ - meta->cols = taosArrayInit(32, POINTER_BYTES); - if (meta->cols == NULL) { - uError("SML:smlBuildSTableMeta failed to allocate memory"); - goto cleanup; +static void smlInsertMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols) { + for (int16_t i = 0; i < taosArrayGetSize(cols); ++i) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &i, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); + } } - return meta; - -cleanup: - taosMemoryFree(meta); - return NULL; } static void smlDestroySTableMeta(SSmlSTableMeta *meta) { @@ -1468,828 +973,245 @@ static void smlDestroySTableMeta(SSmlSTableMeta *meta) { taosMemoryFree(meta); } -static void smlDestroyCols(SArray *cols) { - if (!cols) return; +static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols, bool isTag, SSmlMsgBuf *msg) { for (int i = 0; i < taosArrayGetSize(cols); ++i) { - void *kv = taosArrayGetP(cols, i); - taosMemoryFree(kv); - } -} - -static void smlDestroyInfo(SSmlHandle *info) { - if (!info) return; - qDestroyQuery(info->pQuery); - smlDestroyHandle(info->exec); - - // destroy info->childTables - void **p1 = (void **)taosHashIterate(info->childTables, NULL); - while (p1) { - smlDestroyTableInfo(info, (SSmlTableInfo *)(*p1)); - p1 = (void **)taosHashIterate(info->childTables, p1); - } - taosHashCleanup(info->childTables); - - // destroy info->superTables - p1 = (void **)taosHashIterate(info->superTables, NULL); - while (p1) { - smlDestroySTableMeta((SSmlSTableMeta *)(*p1)); - p1 = (void **)taosHashIterate(info->superTables, p1); - } - taosHashCleanup(info->superTables); - - // destroy info->pVgHash - taosHashCleanup(info->pVgHash); - taosHashCleanup(info->dumplicateKey); - if (!info->dataFormat) { - taosArrayDestroy(info->colsContainer); - } - - cJSON_Delete(info->root); - taosMemoryFreeClear(info); -} - -static SSmlHandle *smlBuildSmlInfo(STscObj *pTscObj, SRequestObj *request, SMLProtocolType protocol, int8_t precision) { - int32_t code = TSDB_CODE_SUCCESS; - SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); - if (NULL == info) { - return NULL; - } - info->id = smlGenId(); - - info->pQuery = (SQuery *)nodesMakeNode(QUERY_NODE_QUERY); - if (NULL == info->pQuery) { - uError("SML:0x%" PRIx64 " create info->pQuery error", info->id); - goto cleanup; - } - info->pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - info->pQuery->haveResultSet = false; - info->pQuery->msgType = TDMT_VND_SUBMIT; - info->pQuery->pRoot = (SNode *)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); - if (NULL == info->pQuery->pRoot) { - uError("SML:0x%" PRIx64 " create info->pQuery->pRoot error", info->id); - goto cleanup; - } - - if (pTscObj) { - info->taos = pTscObj; - code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); - if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); - goto cleanup; - } - } - - info->precision = precision; - info->protocol = protocol; - if (protocol == TSDB_SML_LINE_PROTOCOL) { - info->dataFormat = tsSmlDataFormat; - } else { - info->dataFormat = true; - } - - if (request) { - info->pRequest = request; - info->msgBuf.buf = info->pRequest->msgBuf; - info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; - info->pRequest->stmtType = info->pQuery->pRoot->type; - } - - info->exec = smlInitHandle(info->pQuery); - info->childTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->superTables = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - - info->dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (!info->dataFormat) { - info->colsContainer = taosArrayInit(32, POINTER_BYTES); - if (NULL == info->colsContainer) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } - } - if (NULL == info->exec || NULL == info->childTables || NULL == info->superTables || NULL == info->pVgHash || - NULL == info->dumplicateKey) { - uError("SML:0x%" PRIx64 " create info failed", info->id); - goto cleanup; - } + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); - return info; -cleanup: - smlDestroyInfo(info); - return NULL; -} - -/************* TSDB_SML_JSON_PROTOCOL function start **************/ -static int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo) { - cJSON *metric = cJSON_GetObjectItem(root, "metric"); - if (!cJSON_IsString(metric)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - tinfo->sTableNameLen = strlen(metric->valuestring); - if (IS_INVALID_TABLE_LEN(tinfo->sTableNameLen)) { - uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); - return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - } - - tinfo->sTableName = metric->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int64_t *tsVal) { - int32_t size = cJSON_GetArraySize(root); - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (!cJSON_IsNumber(value)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - double timeDouble = value->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble == 0) { - *tsVal = taosGetTimestampNs(); - return TSDB_CODE_SUCCESS; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - *tsVal = timeDouble; - size_t typeLen = strlen(type->valuestring); - if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { - // seconds - *tsVal = *tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { - switch (type->valuestring[0]) { - case 'm': - case 'M': - // milliseconds - *tsVal = *tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - break; - case 'u': - case 'U': - // microseconds - *tsVal = *tsVal * NANOSECOND_PER_USEC; - timeDouble = timeDouble * NANOSECOND_PER_USEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + int16_t *index = (int16_t *)taosHashGet(metaHash, kv->key, kv->keyLen); + if (index) { + SSmlKv *value = (SSmlKv *)taosArrayGet(metaArray, *index); + if (isTag) { + if (kv->length > value->length) { + value->length = kv->length; } - break; - case 'n': - case 'N': - break; - default: - return TSDB_CODE_TSC_INVALID_JSON; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - return TSDB_CODE_SUCCESS; -} - -static uint8_t smlGetTimestampLen(int64_t num) { - uint8_t len = 0; - while ((num /= 10) != 0) { - len++; - } - len++; - return len; -} - -static int32_t smlParseTSFromJSON(SSmlHandle *info, cJSON *root, SArray *cols) { - // Timestamp must be the first KV to parse - int64_t tsVal = 0; - - cJSON *timestamp = cJSON_GetObjectItem(root, "timestamp"); - if (cJSON_IsNumber(timestamp)) { - // timestamp value 0 indicates current system time - double timeDouble = timestamp->valuedouble; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; - } - - if (timeDouble < 0) { - return TSDB_CODE_INVALID_TIMESTAMP; - } - - uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); - tsVal = (int64_t)timeDouble; - if (tsLen == TSDB_TIME_PRECISION_SEC_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_SEC; - timeDouble = timeDouble * NANOSECOND_PER_SEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + continue; } - } else if (tsLen == TSDB_TIME_PRECISION_MILLI_DIGITS) { - tsVal = tsVal * NANOSECOND_PER_MSEC; - timeDouble = timeDouble * NANOSECOND_PER_MSEC; - if (smlDoubleToInt64OverFlow(timeDouble)) { - smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); - return TSDB_CODE_INVALID_TIMESTAMP; + if (kv->type != value->type) { + smlBuildInvalidDataMsg(msg, "the type is not the same like before", kv->key); + return TSDB_CODE_SML_NOT_SAME_TYPE; } - } else if (timeDouble == 0) { - tsVal = taosGetTimestampNs(); - } else { - return TSDB_CODE_INVALID_TIMESTAMP; - } - } else if (cJSON_IsObject(timestamp)) { - int32_t ret = smlParseTSFromJSONObj(info, timestamp, &tsVal); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Failed to parse timestamp from JSON Obj", info->id); - return ret; - } - } else { - return TSDB_CODE_TSC_INVALID_JSON; - } - - // add ts to - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; - } - kv->key = TS; - kv->keyLen = TS_LEN; - kv->i = tsVal; - kv->type = TSDB_DATA_TYPE_TIMESTAMP; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - taosArrayPush(cols, &kv); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "bool") != 0) { - uError("OTD:invalid type(%s) for JSON Bool", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->type = TSDB_DATA_TYPE_BOOL; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valueint; - - return TSDB_CODE_SUCCESS; -} - -static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { - // tinyint - if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { - if (!IS_VALID_TINYINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_TINYINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // smallint - if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { - if (!IS_VALID_SMALLINT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_SMALLINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // int - if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { - if (!IS_VALID_INT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_INT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->i = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // bigint - if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { - pVal->type = TSDB_DATA_TYPE_BIGINT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - if (value->valuedouble >= (double)INT64_MAX) { - pVal->i = INT64_MAX; - } else if (value->valuedouble <= (double)INT64_MIN) { - pVal->i = INT64_MIN; - } else { - pVal->i = value->valuedouble; - } - return TSDB_CODE_SUCCESS; - } - // float - if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { - if (!IS_VALID_FLOAT(value->valuedouble)) { - uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - pVal->type = TSDB_DATA_TYPE_FLOAT; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->f = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - // double - if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { - pVal->type = TSDB_DATA_TYPE_DOUBLE; - pVal->length = (int16_t)tDataTypes[pVal->type].bytes; - pVal->d = value->valuedouble; - return TSDB_CODE_SUCCESS; - } - - // if reach here means type is unsupported - uError("OTD:invalid type(%s) for JSON Number", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; -} - -static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { - if (strcasecmp(typeStr, "binary") == 0) { - pVal->type = TSDB_DATA_TYPE_BINARY; - } else if (strcasecmp(typeStr, "nchar") == 0) { - pVal->type = TSDB_DATA_TYPE_NCHAR; - } else { - uError("OTD:invalid type(%s) for JSON String", typeStr); - return TSDB_CODE_TSC_INVALID_JSON_TYPE; - } - pVal->length = (int16_t)strlen(value->valuestring); - - if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - if (pVal->type == TSDB_DATA_TYPE_NCHAR && - pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; - } - - pVal->value = value->valuestring; - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { - int32_t ret = TSDB_CODE_SUCCESS; - int32_t size = cJSON_GetArraySize(root); - - if (size != OTD_JSON_SUB_FIELDS_NUM) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *value = cJSON_GetObjectItem(root, "value"); - if (value == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - - cJSON *type = cJSON_GetObjectItem(root, "type"); - if (!cJSON_IsString(type)) { - return TSDB_CODE_TSC_INVALID_JSON; - } - switch (value->type) { - case cJSON_True: - case cJSON_False: { - ret = smlConvertJSONBool(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + if (IS_VAR_DATA_TYPE(kv->type) && (kv->length > value->length)) { // update string len, if bigger + value->length = kv->length; } - break; - } - case cJSON_Number: { - ret = smlConvertJSONNumber(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + } else { + size_t tmp = taosArrayGetSize(metaArray); + if (tmp > INT16_MAX) { + uError("too many cols or tags"); + return -1; } - break; - } - case cJSON_String: { - ret = smlConvertJSONString(kv, type->valuestring, value); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + int16_t size = tmp; + int ret = taosHashPut(metaHash, kv->key, kv->keyLen, &size, SHORT_BYTES); + if (ret == 0) { + taosArrayPush(metaArray, kv); } - break; } - default: - return TSDB_CODE_TSC_INVALID_JSON_TYPE; } return TSDB_CODE_SUCCESS; } -static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { - switch (root->type) { - case cJSON_True: - case cJSON_False: { - kv->type = TSDB_DATA_TYPE_BOOL; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->i = root->valueint; - break; - } - case cJSON_Number: { - kv->type = TSDB_DATA_TYPE_DOUBLE; - kv->length = (int16_t)tDataTypes[kv->type].bytes; - kv->d = root->valuedouble; - break; - } - case cJSON_String: { - /* set default JSON type to binary/nchar according to - * user configured parameter tsDefaultJSONStrType - */ - - char *tsDefaultJSONStrType = "nchar"; // todo - smlConvertJSONString(kv, tsDefaultJSONStrType, root); - break; - } - case cJSON_Object: { - int32_t ret = smlParseValueFromJSONObj(root, kv); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:Failed to parse value from JSON Obj"); - return ret; - } - break; - } - default: - return TSDB_CODE_TSC_INVALID_JSON; +void smlDestroyTableInfo(SSmlHandle *info, SSmlTableInfo *tag) { + for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { + SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); + taosHashCleanup(kvHash); } - return TSDB_CODE_SUCCESS; + // if (info->parseJsonByLib) { + // SSmlLineInfo *key = (SSmlLineInfo *)(tag->key); + // if (key != NULL) taosMemoryFree(key->tags); + // } + // taosMemoryFree(tag->key); + taosArrayDestroy(tag->cols); + taosArrayDestroy(tag->tags); + taosMemoryFree(tag); } -static int32_t smlParseColsFromJSON(cJSON *root, SArray *cols) { - if (!cols) return TSDB_CODE_OUT_OF_MEMORY; - cJSON *metricVal = cJSON_GetObjectItem(root, "value"); - if (metricVal == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; +void clearColValArray(SArray *pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal *pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } } +} - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) { - return TSDB_CODE_OUT_OF_MEMORY; +void smlDestroyInfo(SSmlHandle *info) { + if (!info) return; + qDestroyQuery(info->pQuery); + + // destroy info->childTables + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); + while (oneTable) { + smlDestroyTableInfo(info, *oneTable); + oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } - taosArrayPush(cols, &kv); - kv->key = VALUE; - kv->keyLen = VALUE_LEN; - int32_t ret = smlParseValueFromJSON(metricVal, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; + // destroy info->superTables + SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); + while (oneSTable) { + smlDestroySTableMeta(*oneSTable); + oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); } - return TSDB_CODE_SUCCESS; -} -static int32_t smlParseTagsFromJSON(cJSON *root, SArray *pKVs, char *childTableName, SHashObj *dumplicateKey, - SSmlMsgBuf *msg) { - int32_t ret = TSDB_CODE_SUCCESS; - if (!pKVs) { - return TSDB_CODE_OUT_OF_MEMORY; + // destroy info->pVgHash + taosHashCleanup(info->pVgHash); + taosHashCleanup(info->childTables); + taosHashCleanup(info->superTables); + + for (int i = 0; i < taosArrayGetSize(info->tagJsonArray); i++) { + cJSON *tags = (cJSON *)taosArrayGetP(info->tagJsonArray, i); + cJSON_Delete(tags); } - cJSON *tags = cJSON_GetObjectItem(root, "tags"); - if (tags == NULL || tags->type != cJSON_Object) { - return TSDB_CODE_TSC_INVALID_JSON; + taosArrayDestroy(info->tagJsonArray); + + for (int i = 0; i < taosArrayGetSize(info->valueJsonArray); i++) { + cJSON *value = (cJSON *)taosArrayGetP(info->valueJsonArray, i); + cJSON_Delete(value); } + taosArrayDestroy(info->valueJsonArray); - size_t childTableNameLen = strlen(tsSmlChildTableName); - int32_t tagNum = cJSON_GetArraySize(tags); - for (int32_t i = 0; i < tagNum; ++i) { - cJSON *tag = cJSON_GetArrayItem(tags, i); - if (tag == NULL) { - return TSDB_CODE_TSC_INVALID_JSON; - } - size_t keyLen = strlen(tag->string); - if (IS_INVALID_COL_LEN(keyLen)) { - uError("OTD:Tag key length is 0 or too large than 64"); - return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; - } - // check duplicate keys - if (smlCheckDuplicateKey(tag->string, keyLen, dumplicateKey)) { - return TSDB_CODE_TSC_DUP_NAMES; - } + taosArrayDestroy(info->preLineTagKV); + taosArrayDestroy(info->maxTagKVs); + taosArrayDestroy(info->preLineColKV); - // handle child table name - if (childTableNameLen != 0 && strcmp(tag->string, tsSmlChildTableName) == 0) { - if (!cJSON_IsString(tag)) { - uError("OTD:ID must be JSON string"); - return TSDB_CODE_TSC_INVALID_JSON; + if (!info->dataFormat) { + for (int i = 0; i < info->lineNum; i++) { + taosArrayDestroy(info->lines[i].colArray); + if (info->parseJsonByLib) { + taosMemoryFree(info->lines[i].tags); } - memset(childTableName, 0, TSDB_TABLE_NAME_LEN); - tstrncpy(childTableName, tag->valuestring, TSDB_TABLE_NAME_LEN); - continue; + if (info->lines[i].measureTagsLen != 0) taosMemoryFree(info->lines[i].measureTag); } + taosMemoryFree(info->lines); + } - // add kv to SSmlKv - SSmlKv *kv = (SSmlKv *)taosMemoryCalloc(sizeof(SSmlKv), 1); - if (!kv) return TSDB_CODE_OUT_OF_MEMORY; - taosArrayPush(pKVs, &kv); - - // key - kv->keyLen = keyLen; - kv->key = tag->string; + cJSON_Delete(info->root); + taosMemoryFreeClear(info); +} - // value - ret = smlParseValueFromJSON(tag, kv); - if (ret != TSDB_CODE_SUCCESS) { - return ret; +SSmlHandle *smlBuildSmlInfo(TAOS *taos) { + int32_t code = TSDB_CODE_SUCCESS; + SSmlHandle *info = (SSmlHandle *)taosMemoryCalloc(1, sizeof(SSmlHandle)); + if (NULL == info) { + return NULL; + } + if (taos != NULL) { + info->taos = acquireTscObj(*(int64_t *)taos); + code = catalogGetHandle(info->taos->pAppInfo->clusterId, &info->pCatalog); + if (code != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " get catalog error %d", info->id, code); + goto cleanup; } } - return ret; -} + info->pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + info->childTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + info->superTables = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); -static int32_t smlParseJSONString(SSmlHandle *info, cJSON *root, SSmlTableInfo *tinfo, SArray *cols) { - int32_t ret = TSDB_CODE_SUCCESS; + info->id = smlGenId(); + info->pQuery = smlInitHandle(); + info->dataFormat = true; - if (!cJSON_IsObject(root)) { - uError("OTD:0x%" PRIx64 " data point needs to be JSON object", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } + info->tagJsonArray = taosArrayInit(8, POINTER_BYTES); + info->valueJsonArray = taosArrayInit(8, POINTER_BYTES); + info->preLineTagKV = taosArrayInit(8, sizeof(SSmlKv)); + info->maxTagKVs = taosArrayInit(8, sizeof(SSmlKv)); + info->preLineColKV = taosArrayInit(8, sizeof(SSmlKv)); - int32_t size = cJSON_GetArraySize(root); - // outmost json fields has to be exactly 4 - if (size != OTD_JSON_FIELDS_NUM) { - uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); - return TSDB_CODE_TSC_INVALID_JSON; + if (NULL == info->pVgHash || NULL == info->childTables || NULL == info->superTables) { + uError("create SSmlHandle failed"); + goto cleanup; } - // Parse metric - ret = smlParseMetricFromJSON(info, root, tinfo); - if (ret != TSDB_CODE_SUCCESS) { - uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse metric from JSON payload finished", info->id); + return info; - // Parse timestamp - ret = smlParseTSFromJSON(info, root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); - return ret; - } - uDebug("OTD:0x%" PRIx64 " Parse timestamp from JSON payload finished", info->id); +cleanup: + smlDestroyInfo(info); + return NULL; +} - // Parse metric value - ret = smlParseColsFromJSON(root, cols); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); - return ret; +static int32_t smlPushCols(SArray *colsArray, SArray *cols) { + SHashObj *kvHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (!kvHash) { + uError("SML:smlDealCols failed to allocate memory"); + return TSDB_CODE_OUT_OF_MEMORY; } - uDebug("OTD:0x%" PRIx64 " Parse metric value from JSON payload finished", info->id); - - // Parse tags - ret = smlParseTagsFromJSON(root, tinfo->tags, tinfo->childTableName, info->dumplicateKey, &info->msgBuf); - if (ret) { - uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); - return ret; + for (size_t i = 0; i < taosArrayGetSize(cols); i++) { + SSmlKv *kv = (SSmlKv *)taosArrayGet(cols, i); + taosHashPut(kvHash, kv->key, kv->keyLen, &kv, POINTER_BYTES); } - uDebug("OTD:0x%" PRIx64 " Parse tags from JSON payload finished", info->id); + taosArrayPush(colsArray, &kvHash); return TSDB_CODE_SUCCESS; } -/************* TSDB_SML_JSON_PROTOCOL function end **************/ - -static int32_t smlParseInfluxLine(SSmlHandle *info, const char *sql, const int len) { - SSmlLineInfo elements = {0}; - uDebug("SML:0x%" PRIx64 " smlParseInfluxLine raw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, (info->isRawLine ? "rawdata" : sql)); - int ret = smlParseInfluxString(sql, sql + len, &elements, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed", info->id); - return ret; - } +static int32_t smlParseLineBottom(SSmlHandle *info) { + if (info->dataFormat) return TSDB_CODE_SUCCESS; - SArray *cols = NULL; - if (info->dataFormat) { // if dataFormat, cols need new memory to save data - cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseInfluxLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - } else { // if dataFormat is false, cols do not need to save data, there is another new memory to save data - cols = info->colsContainer; - } - - ret = smlParseTS(info, elements.timestamp, elements.timestampLen, cols); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTS failed", info->id); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - ret = smlParseCols(elements.cols, elements.colsLen, cols, NULL, false, info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse cloums fields failed", info->id); - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return ret; - } - - bool hasTable = true; - SSmlTableInfo *tinfo = NULL; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, elements.measure, elements.measureTagsLen); - if (!oneTable) { - tinfo = smlBuildTableInfo(); - if (!tinfo) { - smlDestroyCols(cols); - if (info->dataFormat) taosArrayDestroy(cols); - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < info->lineNum; i++) { + SSmlLineInfo *elements = info->lines + i; + SSmlTableInfo *tinfo = NULL; + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { + tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measure, elements->measureTagsLen); + } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { + tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, + elements->measureLen + elements->tagsLen); + } else { + tinfo = *(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, + elements->measureLen + elements->tagsLen); } - taosHashPut(info->childTables, elements.measure, elements.measureTagsLen, &tinfo, POINTER_BYTES); - oneTable = &tinfo; - hasTable = false; - } - - ret = smlDealCols(*oneTable, info->dataFormat, cols); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } - if (!hasTable) { - ret = smlParseCols(elements.tags, elements.tagsLen, (*oneTable)->tags, (*oneTable)->childTableName, true, - info->dumplicateKey, &info->msgBuf); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseCols parse tag fields failed", info->id); - return ret; + if (tinfo == NULL) { + uError("SML:0x%" PRIx64 "get oneTable failed, line num:%d", info->id, i); + smlBuildInvalidDataMsg(&info->msgBuf, "get oneTable failed", elements->measure); + return TSDB_CODE_SML_INVALID_DATA; } - if (taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_TAGS) { + if (taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many tags than 128", NULL); return TSDB_CODE_PAR_INVALID_TAGS_NUM; } - if (taosArrayGetSize(cols) + taosArrayGetSize((*oneTable)->tags) > TSDB_MAX_COLUMNS) { + if (taosArrayGetSize(elements->colArray) + taosArrayGetSize(tinfo->tags) > TSDB_MAX_COLUMNS) { smlBuildInvalidDataMsg(&info->msgBuf, "too many columns than 4096", NULL); return TSDB_CODE_PAR_TOO_MANY_COLUMNS; } - (*oneTable)->sTableName = elements.measure; - (*oneTable)->sTableNameLen = elements.measureLen; - if (strlen((*oneTable)->childTableName) == 0) { - RandTableName rName = {(*oneTable)->tags, (*oneTable)->sTableName, (uint8_t)(*oneTable)->sTableNameLen, - (*oneTable)->childTableName}; - - buildChildTableName(&rName); - } - (*oneTable)->uid = info->uid++; - } - - SSmlSTableMeta **tableMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, elements.measure, elements.measureLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } + int ret = smlPushCols(tinfo->cols, elements->colArray); if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); return ret; } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, elements.measure, elements.measureLen, &meta, POINTER_BYTES); - } - - if (!info->dataFormat) { - taosArrayClear(info->colsContainer); - } - taosHashClear(info->dumplicateKey); - return TSDB_CODE_SUCCESS; -} - -static int32_t smlParseTelnetLine(SSmlHandle *info, void *data, const int len) { - int ret = TSDB_CODE_SUCCESS; - SSmlTableInfo *tinfo = smlBuildTableInfo(); - if (!tinfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - if (cols == NULL) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed to allocate memory", info->id); - return TSDB_CODE_OUT_OF_MEMORY; - } - - if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - ret = smlParseTelnetString(info, (const char *)data, (char *)data + len, tinfo, cols); - } else if (info->protocol == TSDB_SML_JSON_PROTOCOL) { - ret = smlParseJSONString(info, (cJSON *)data, tinfo, cols); - } else { - ASSERT(0); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlParseTelnetLine failed", info->id); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return ret; - } - - if (taosArrayGetSize(tinfo->tags) <= 0 || taosArrayGetSize(tinfo->tags) > TSDB_MAX_TAGS) { - smlBuildInvalidDataMsg(&info->msgBuf, "invalidate tags length:[1,128]", NULL); - smlDestroyTableInfo(info, tinfo); - smlDestroyCols(cols); - taosArrayDestroy(cols); - return TSDB_CODE_PAR_INVALID_TAGS_NUM; - } - taosHashClear(info->dumplicateKey); - - if (strlen(tinfo->childTableName) == 0) { - RandTableName rName = {tinfo->tags, tinfo->sTableName, (uint8_t)tinfo->sTableNameLen, tinfo->childTableName}; - buildChildTableName(&rName); - } - bool hasTable = true; - SSmlTableInfo **oneTable = - (SSmlTableInfo **)taosHashGet(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName)); - if (!oneTable) { - taosHashPut(info->childTables, tinfo->childTableName, strlen(tinfo->childTableName), &tinfo, POINTER_BYTES); - oneTable = &tinfo; - tinfo->uid = info->uid++; - hasTable = false; - } else { - smlDestroyTableInfo(info, tinfo); - } + SSmlSTableMeta **tableMeta = + (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + if (tableMeta) { // update meta + ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, elements->colArray, false, &info->msgBuf); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, tinfo->tags, true, &info->msgBuf); + } + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + return ret; + } + } else { + // ret = smlJudgeDupColName(elements->colArray, tinfo->tags, &info->msgBuf); + // if (ret != TSDB_CODE_SUCCESS) { + // uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); + // return ret; + // } - taosArrayPush((*oneTable)->cols, &cols); - SSmlSTableMeta **tableMeta = - (SSmlSTableMeta **)taosHashGet(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen); - if (tableMeta) { // update meta - ret = smlUpdateMeta((*tableMeta)->colHash, (*tableMeta)->cols, cols, &info->msgBuf); - if (!hasTable && ret == TSDB_CODE_SUCCESS) { - ret = smlUpdateMeta((*tableMeta)->tagHash, (*tableMeta)->tags, (*oneTable)->tags, &info->msgBuf); - } - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlUpdateMeta failed", info->id); - return ret; + SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); + smlInsertMeta(meta->tagHash, meta->tags, tinfo->tags); + smlInsertMeta(meta->colHash, meta->cols, elements->colArray); + taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); } - } else { - SSmlSTableMeta *meta = smlBuildSTableMeta(); - smlInsertMeta(meta->tagHash, meta->tags, (*oneTable)->tags); - smlInsertMeta(meta->colHash, meta->cols, cols); - taosHashPut(info->superTables, (*oneTable)->sTableName, (*oneTable)->sTableNameLen, &meta, POINTER_BYTES); } return TSDB_CODE_SUCCESS; } -static int32_t smlParseJSON(SSmlHandle *info, char *payload) { - int32_t payloadNum = 0; - int32_t ret = TSDB_CODE_SUCCESS; - - if (payload == NULL) { - uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); - return TSDB_CODE_TSC_INVALID_JSON; - } - - info->root = cJSON_Parse(payload); - if (info->root == NULL) { - uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); - return TSDB_CODE_TSC_INVALID_JSON; - } - // multiple data points must be sent in JSON array - if (cJSON_IsObject(info->root)) { - payloadNum = 1; - } else if (cJSON_IsArray(info->root)) { - payloadNum = cJSON_GetArraySize(info->root); - } else { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - ret = TSDB_CODE_TSC_INVALID_JSON; - goto end; - } - - for (int32_t i = 0; i < payloadNum; ++i) { - cJSON *dataPoint = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : cJSON_GetArrayItem(info->root, i); - ret = smlParseTelnetLine(info, dataPoint, -1); - if (ret != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " Invalid JSON Payload", info->id); - goto end; - } - } - -end: - return ret; -} - static int32_t smlInsertData(SSmlHandle *info) { int32_t code = TSDB_CODE_SUCCESS; @@ -2317,13 +1239,16 @@ static int32_t smlInsertData(SSmlHandle *info) { SSmlSTableMeta **pMeta = (SSmlSTableMeta **)taosHashGet(info->superTables, tableData->sTableName, tableData->sTableNameLen); - ASSERT(NULL != pMeta && NULL != *pMeta); + if (unlikely(NULL == pMeta || NULL == (*pMeta)->tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == pMeta. table name: %s", info->id, tableData->childTableName); + return TSDB_CODE_SML_INTERNAL_ERROR; + } // use tablemeta of stable to save vgid and uid of child table (*pMeta)->tableMeta->vgId = vg.vgId; (*pMeta)->tableMeta->uid = tableData->uid; // one table merge data block together according uid - code = smlBindData(info->exec, tableData->tags, (*pMeta)->cols, tableData->cols, info->dataFormat, + code = smlBindData(info->pQuery, info->dataFormat, tableData->tags, (*pMeta)->cols, tableData->cols, (*pMeta)->tableMeta, tableData->childTableName, tableData->sTableName, tableData->sTableNameLen, info->ttl, info->msgBuf.buf, info->msgBuf.len); if (code != TSDB_CODE_SUCCESS) { @@ -2333,7 +1258,7 @@ static int32_t smlInsertData(SSmlHandle *info) { oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); } - code = smlBuildOutput(info->exec, info->pVgHash); + code = smlBuildOutput(info->pQuery, info->pVgHash); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildOutput failed", info->id); return code; @@ -2348,15 +1273,53 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - uError("SML:0x%" PRIx64 - " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ + uDebug( + "SML:0x%" PRIx64 + " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 - "", - info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, - info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, - info->cost.schemaTime - info->cost.parseTime, - info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, - info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); + "", + info->id, info->cost.code, info->cost.lineNum, info->cost.numOfSTables, info->cost.numOfCTables, + info->cost.numOfCreateSTables, info->cost.numOfAlterTagSTables, info->cost.numOfAlterColSTables, + info->cost.schemaTime - info->cost.parseTime, info->cost.insertBindTime - info->cost.schemaTime, + info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.endTime - info->cost.insertRpcTime, + info->cost.endTime - info->cost.parseTime); +} + +int32_t smlClearForRerun(SSmlHandle *info) { + info->reRun = false; + // clear info->childTables + SSmlTableInfo **oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, NULL); + while (oneTable) { + smlDestroyTableInfo(info, *oneTable); + oneTable = (SSmlTableInfo **)taosHashIterate(info->childTables, oneTable); + } + + // clear info->superTables + SSmlSTableMeta **oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, NULL); + while (oneSTable) { + smlDestroySTableMeta(*oneSTable); + oneSTable = (SSmlSTableMeta **)taosHashIterate(info->superTables, oneSTable); + } + + taosHashClear(info->childTables); + taosHashClear(info->superTables); + + if (!info->dataFormat) { + if (unlikely(info->lines != NULL)) { + uError("SML:0x%" PRIx64 " info->lines != NULL", info->id); + return TSDB_CODE_SML_INVALID_DATA; + } + info->lines = (SSmlLineInfo *)taosMemoryCalloc(info->lineNum, sizeof(SSmlLineInfo)); + } + + memset(&info->preLine, 0, sizeof(SSmlLineInfo)); + info->currSTableMeta = NULL; + info->currTableDataCtx = NULL; + + SVnodeModifyOpStmt *stmt = (SVnodeModifyOpStmt *)(info->pQuery->pRoot); + stmt->freeHashFunc(stmt->pTableBlockHashObj); + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + return TSDB_CODE_SUCCESS; } static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char *rawLineEnd, int numLines) { @@ -2374,7 +1337,9 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char return code; } - for (int32_t i = 0; i < numLines; ++i) { + char *oldRaw = rawLine; + int32_t i = 0; + while (i < numLines) { char *tmp = NULL; int len = 0; if (lines) { @@ -2393,18 +1358,43 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char } } + uDebug("SML:0x%" PRIx64 " smlParseLine israw:%d, len:%d, sql:%s", info->id, info->isRawLine, len, + (info->isRawLine ? "rawdata" : tmp)); + if (info->protocol == TSDB_SML_LINE_PROTOCOL) { - code = smlParseInfluxLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseInfluxString(info, tmp, tmp + len, &element); + } else { + code = smlParseInfluxString(info, tmp, tmp + len, info->lines + i); + } } else if (info->protocol == TSDB_SML_TELNET_PROTOCOL) { - code = smlParseTelnetLine(info, tmp, len); + if (info->dataFormat) { + SSmlLineInfo element = {0}; + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, &element); + if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); + } else { + code = smlParseTelnetString(info, (char *)tmp, (char *)tmp + len, info->lines + i); + } } else { - ASSERT(0); + code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; } if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); return code; } + if (info->reRun) { + i = 0; + rawLine = oldRaw; + code = smlClearForRerun(info); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + continue; + } + i++; } + return code; } @@ -2419,8 +1409,13 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL uError("SML:0x%" PRIx64 " smlParseLine error : %s", info->id, tstrerror(code)); return code; } + code = smlParseLineBottom(info); + if (code != 0) { + uError("SML:0x%" PRIx64 " smlParseLineBottom error : %s", info->id, tstrerror(code)); + return code; + } - info->cost.lineNum = numLines; + info->cost.lineNum = info->lineNum; info->cost.numOfSTables = taosHashGetSize(info->superTables); info->cost.numOfCTables = taosHashGetSize(info->childTables); @@ -2446,57 +1441,42 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } -static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { - // SCatalog *catalog = NULL; - // int32_t code = catalogGetHandle(((STscObj *)taos)->pAppInfo->clusterId, &catalog); - // if (code != TSDB_CODE_SUCCESS) { - // uError("SML get catalog error %d", code); - // return code; - // } - // - // SName name; - // tNameSetDbName(&name, taos->acctId, taos->db, strlen(taos->db)); - // char dbFname[TSDB_DB_FNAME_LEN] = {0}; - // tNameGetFullDbName(&name, dbFname); - // SDbCfgInfo pInfo = {0}; - // - // SRequestConnInfo conn = {0}; - // conn.pTrans = taos->pAppInfo->pTransporter; - // conn.requestId = request->requestId; - // conn.requestObjRefId = request->self; - // conn.mgmtEps = getEpSet_s(&taos->pAppInfo->mgmtEp); - // - // code = catalogGetDBCfg(catalog, &conn, dbFname, &pInfo); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // taosArrayDestroy(pInfo.pRetensions); - // - // if (!pInfo.schemaless) { - // return TSDB_CODE_SML_INVALID_DB_CONF; - // } - return TSDB_CODE_SUCCESS; -} +TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, + int protocol, int precision, int32_t ttl, int64_t reqid) { + int32_t code = TSDB_CODE_SUCCESS; + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } -TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd, - int numLines, int protocol, int precision, int32_t ttl) { - STscObj *pTscObj = request->pTscObj; - SSmlHandle *info = NULL; - pTscObj->schemalessType = 1; - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; + SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); + if (request == NULL) { + uError("SML:taos_schemaless_insert error request is null"); + return NULL; + } + SSmlHandle *info = smlBuildSmlInfo(taos); + if (info == NULL) { + request->code = TSDB_CODE_OUT_OF_MEMORY; + uError("SML:taos_schemaless_insert error SSmlHandle is null"); + return (TAOS_RES *)request; + } + info->pRequest = request; + info->isRawLine = rawLine != NULL; + info->ttl = ttl; + info->precision = precision; + info->protocol = (TSDB_SML_PROTOCOL_TYPE)protocol; + info->msgBuf.buf = info->pRequest->msgBuf; + info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; + info->lineNum = numLines; + + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; smlBuildInvalidDataMsg(&msg, "Database not specified", NULL); goto end; } - if (isSchemalessDb(pTscObj, request) != TSDB_CODE_SUCCESS) { - request->code = TSDB_CODE_SML_INVALID_DB_CONF; - smlBuildInvalidDataMsg(&msg, "Cannot write data to a non schemaless database", NULL); - goto end; - } - if (protocol < TSDB_SML_LINE_PROTOCOL || protocol > TSDB_SML_JSON_PROTOCOL) { request->code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; smlBuildInvalidDataMsg(&msg, "protocol invalidate", NULL); @@ -2518,24 +1498,14 @@ TAOS_RES *taos_schemaless_insert_inner(SRequestObj *request, char *lines[], char goto end; } - info = smlBuildSmlInfo(pTscObj, request, (SMLProtocolType)protocol, precision); - if (!info) { - request->code = TSDB_CODE_OUT_OF_MEMORY; - uError("SML:taos_schemaless_insert error SSmlHandle is null"); - goto end; - } - - info->isRawLine = (rawLine == NULL); - info->ttl = ttl; - request->code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); - -end: - uDebug("SML:0x%" PRIx64 " insert finished, code: %d", info->id, request->code); + code = smlProcess(info, lines, rawLine, rawLineEnd, numLines); + request->code = code; info->cost.endTime = taosGetTimestampUs(); - info->cost.code = request->code; + info->cost.code = code; smlPrintStatisticInfo(info); - smlDestroyInfo(info); +end: + smlDestroyInfo(info); return (TAOS_RES *)request; } @@ -2560,59 +1530,26 @@ end: TAOS_RES *taos_schemaless_insert_ttl_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - - return taos_schemaless_insert_inner(request, lines, NULL, NULL, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, lines, NULL, NULL, numLines, protocol, precision, ttl, reqid); } TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) { return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); } -TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int32_t ttl) { +TAOS_RES *taos_schemaless_insert_ttl(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int32_t ttl) { return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid) { - return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, + int64_t reqid) { + return taos_schemaless_insert_ttl_with_reqid(taos, lines, numLines, protocol, precision, TSDB_DEFAULT_TABLE_TTL, + reqid); } TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, - int precision, int32_t ttl, int64_t reqid) { - if (NULL == taos) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - SRequestObj *request = (SRequestObj *)createRequest(*(int64_t *)taos, TSDB_SQL_INSERT, reqid); - if (!request) { - uError("SML:taos_schemaless_insert error request is null"); - return NULL; - } - - if (!lines || len <= 0) { - SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - request->code = TSDB_CODE_SML_INVALID_DATA; - smlBuildInvalidDataMsg(&msg, "lines is null", NULL); - return (TAOS_RES *)request; - } - + int precision, int32_t ttl, int64_t reqid) { int numLines = 0; *totalRows = 0; char *tmp = lines; @@ -2625,16 +1562,21 @@ TAOS_RES *taos_schemaless_insert_raw_ttl_with_reqid(TAOS *taos, char *lines, int tmp = lines + i + 1; } } - return taos_schemaless_insert_inner(request, NULL, lines, lines + len, numLines, protocol, precision, ttl); + return taos_schemaless_insert_inner(taos, NULL, lines, lines + len, *totalRows, protocol, precision, ttl, reqid); } -TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid) { - return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, reqid); +TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int64_t reqid) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, + TSDB_DEFAULT_TABLE_TTL, reqid); } -TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int32_t ttl) { +TAOS_RES *taos_schemaless_insert_raw_ttl(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision, int32_t ttl) { return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, ttl, 0); } -TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision) { - return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, TSDB_DEFAULT_TABLE_TTL, 0); +TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, + int precision) { + return taos_schemaless_insert_raw_ttl_with_reqid(taos, lines, len, totalRows, protocol, precision, + TSDB_DEFAULT_TABLE_TTL, 0); } diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c new file mode 100644 index 0000000000000000000000000000000000000000..e89227d4123e5f071edbb24795d93889c8bc0e68 --- /dev/null +++ b/source/client/src/clientSmlJson.c @@ -0,0 +1,1312 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "clientSml.h" + +#define OTD_JSON_SUB_FIELDS_NUM 2 + +#define JUMP_JSON_SPACE(start) \ + while (*(start)) { \ + if (unlikely(*(start) > 32)) \ + break; \ + else \ + (start)++; \ + } + +// SArray *smlJsonParseTags(char *start, char *end){ +// SArray *tags = taosArrayInit(4, sizeof(SSmlKv)); +// while(start < end){ +// SSmlKv kv = {0}; +// kv.type = TSDB_DATA_TYPE_NCHAR; +// bool isInQuote = false; +// while(start < end){ +// if(unlikely(!isInQuote && *start == '"')){ +// start++; +// kv.key = start; +// isInQuote = true; +// continue; +// } +// if(unlikely(isInQuote && *start == '"')){ +// kv.keyLen = start - kv.key; +// start++; +// break; +// } +// start++; +// } +// bool hasColon = false; +// while(start < end){ +// if(unlikely(!hasColon && *start == ':')){ +// start++; +// hasColon = true; +// continue; +// } +// if(unlikely(hasColon && kv.value == NULL && (*start > 32 && *start != '"'))){ +// kv.value = start; +// start++; +// continue; +// } +// +// if(unlikely(hasColon && kv.value != NULL && (*start == '"' || *start == ',' || *start == '}'))){ +// kv.length = start - kv.value; +// taosArrayPush(tags, &kv); +// start++; +// break; +// } +// start++; +// } +// } +// return tags; +// } + +// static int32_t smlParseTagsFromJSON(SSmlHandle *info, SSmlLineInfo *elements) { +// int32_t ret = TSDB_CODE_SUCCESS; +// +// if(is_same_child_table_telnet(elements, &info->preLine) == 0){ +// return TSDB_CODE_SUCCESS; +// } +// +// bool isSameMeasure = IS_SAME_SUPER_TABLE; +// +// int cnt = 0; +// SArray *preLineKV = info->preLineTagKV; +// bool isSuperKVInit = true; +// SArray *superKV = NULL; +// if(info->dataFormat){ +// if(unlikely(!isSameMeasure)){ +// SSmlSTableMeta *sMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, +// elements->measureLen, NULL); +// +// if(unlikely(sMeta == NULL)){ +// sMeta = smlBuildSTableMeta(info->dataFormat); +// STableMeta * pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); +// sMeta->tableMeta = pTableMeta; +// if(pTableMeta == NULL){ +// info->dataFormat = false; +// info->reRun = true; +// return TSDB_CODE_SUCCESS; +// } +// nodeListSet(&info->superTables, elements->measure, elements->measureLen, sMeta, NULL); +// } +// info->currSTableMeta = sMeta->tableMeta; +// superKV = sMeta->tags; +// +// if(unlikely(taosArrayGetSize(superKV) == 0)){ +// isSuperKVInit = false; +// } +// taosArraySetSize(preLineKV, 0); +// } +// }else{ +// taosArraySetSize(preLineKV, 0); +// } +// +// SArray *tags = smlJsonParseTags(elements->tags, elements->tags + elements->tagsLen); +// int32_t tagNum = taosArrayGetSize(tags); +// if (tagNum == 0) { +// uError("SML:tag is empty:%s", elements->tags) +// taosArrayDestroy(tags); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// for (int32_t i = 0; i < tagNum; ++i) { +// SSmlKv kv = *(SSmlKv*)taosArrayGet(tags, i); +// +// if(info->dataFormat){ +// if(unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// +// if(isSameMeasure){ +// if(unlikely(cnt >= taosArrayGetSize(preLineKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); +// if(unlikely(kv.length > preKV->length)){ +// preKV->length = kv.length; +// SSmlSTableMeta *tableMeta = (SSmlSTableMeta *)nodeListGet(info->superTables, elements->measure, +// elements->measureLen, NULL); +// if(unlikely(NULL == tableMeta)){ +// uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); +// return TSDB_CODE_SML_INTERNAL_ERROR; +// } +// +// SSmlKv *oldKV = (SSmlKv *)taosArrayGet(tableMeta->tags, cnt); +// oldKV->length = kv.length; +// info->needModifySchema = true; +// } +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// if(isSuperKVInit){ +// if(unlikely(cnt >= taosArrayGetSize(superKV))) { +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// SSmlKv *preKV = (SSmlKv *)taosArrayGet(superKV, cnt); +// if(unlikely(kv.length > preKV->length)) { +// preKV->length = kv.length; +// }else{ +// kv.length = preKV->length; +// } +// info->needModifySchema = true; +// +// if(unlikely(!IS_SAME_KEY)){ +// info->dataFormat = false; +// info->reRun = true; +// taosArrayDestroy(tags); +// return TSDB_CODE_SUCCESS; +// } +// }else{ +// taosArrayPush(superKV, &kv); +// } +// taosArrayPush(preLineKV, &kv); +// } +// }else{ +// taosArrayPush(preLineKV, &kv); +// } +// cnt++; +// } +// taosArrayDestroy(tags); +// +// SSmlTableInfo *tinfo = (SSmlTableInfo *)nodeListGet(info->childTables, elements, POINTER_BYTES, +// is_same_child_table_telnet); if (unlikely(tinfo == NULL)) { +// tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); +// if (unlikely(!tinfo)) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } +// tinfo->tags = taosArrayDup(preLineKV, NULL); +// +// smlSetCTableName(tinfo); +// if (info->dataFormat) { +// info->currSTableMeta->uid = tinfo->uid; +// tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); +// if (tinfo->tableDataCtx == NULL) { +// smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); +// return TSDB_CODE_SML_INVALID_DATA; +// } +// } +// +// SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); +// *key = *elements; +// tinfo->key = key; +// nodeListSet(&info->childTables, key, POINTER_BYTES, tinfo, is_same_child_table_telnet); +// } +// if (info->dataFormat) info->currTableDataCtx = tinfo->tableDataCtx; +// +// return ret; +// } + +static char *smlJsonGetObj(char *payload) { + int leftBracketCnt = 0; + bool isInQuote = false; + while (*payload) { + if (*payload == '"' && *(payload - 1) != '\\') { + isInQuote = !isInQuote; + } else if (!isInQuote && unlikely(*payload == '{')) { + leftBracketCnt++; + payload++; + continue; + } else if (!isInQuote && unlikely(*payload == '}')) { + leftBracketCnt--; + payload++; + if (leftBracketCnt == 0) { + return payload; + } else if (leftBracketCnt < 0) { + return NULL; + } + continue; + } + payload++; + } + return NULL; +} + +int smlJsonParseObjFirst(char **start, SSmlLineInfo *element, int8_t *offset) { + int index = 0; + while (*(*start)) { + if ((*start)[0] != '"') { + (*start)++; + continue; + } + + if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + } + + char *sTmp = *start; + if ((*start)[1] == 'm' && (*start)[2] == 'e' && (*start)[3] == 't' && (*start)[4] == 'r' && (*start)[5] == 'i' && + (*start)[6] == 'c' && (*start)[7] == '"') { + (*start) += 8; + bool isInQuote = false; + while (*(*start)) { + if (unlikely(!isInQuote && *(*start) == '"')) { + (*start)++; + offset[index++] = *start - sTmp; + element->measure = (*start); + isInQuote = true; + continue; + } + if (unlikely(isInQuote && *(*start) == '"')) { + element->measureLen = (*start) - element->measure; + (*start)++; + break; + } + (*start)++; + } + } else if ((*start)[1] == 't' && (*start)[2] == 'i' && (*start)[3] == 'm' && (*start)[4] == 'e' && + (*start)[5] == 's' && (*start)[6] == 't' && (*start)[7] == 'a' && (*start)[8] == 'm' && + (*start)[9] == 'p' && (*start)[10] == '"') { + (*start) += 11; + bool hasColon = false; + while (*(*start)) { + if (unlikely(!hasColon && *(*start) == ':')) { + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->timestamp = (*start); + if (*(*start) == '{') { + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->timestampLen = tmp - (*start); + *start = tmp; + } + break; + } + hasColon = true; + continue; + } + if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + } else if ((*start)[1] == 'v' && (*start)[2] == 'a' && (*start)[3] == 'l' && (*start)[4] == 'u' && + (*start)[5] == 'e' && (*start)[6] == '"') { + (*start) += 7; + + bool hasColon = false; + while (*(*start)) { + if (unlikely(!hasColon && *(*start) == ':')) { + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->cols = (*start); + if (*(*start) == '{') { + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->colsLen = tmp - (*start); + *start = tmp; + } + break; + } + hasColon = true; + continue; + } + if (unlikely(hasColon && (*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32))) { + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + } else if ((*start)[1] == 't' && (*start)[2] == 'a' && (*start)[3] == 'g' && (*start)[4] == 's' && + (*start)[5] == '"') { + (*start) += 6; + + while (*(*start)) { + if (unlikely(*(*start) == ':')) { + (*start)++; + JUMP_JSON_SPACE((*start)) + offset[index++] = *start - sTmp; + element->tags = (*start); + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->tagsLen = tmp - (*start); + *start = tmp; + } + break; + } + (*start)++; + } + } + if (*(*start) == '\0') { + break; + } + if (*(*start) == '}') { + (*start)++; + break; + } + (*start)++; + } + + if (unlikely(index != OTD_JSON_FIELDS_NUM) || element->tags == NULL || element->cols == NULL || + element->measure == NULL || element->timestamp == NULL) { + uError("elements != %d or element parse null", OTD_JSON_FIELDS_NUM) return -1; + } + return 0; +} + +int smlJsonParseObj(char **start, SSmlLineInfo *element, int8_t *offset) { + int index = 0; + while (*(*start)) { + if ((*start)[0] != '"') { + (*start)++; + continue; + } + + if (unlikely(index >= OTD_JSON_FIELDS_NUM)) { + uError("index >= %d, %s", OTD_JSON_FIELDS_NUM, *start) return -1; + } + + if ((*start)[1] == 'm') { + (*start) += offset[index++]; + element->measure = *start; + while (*(*start)) { + if (unlikely(*(*start) == '"')) { + element->measureLen = (*start) - element->measure; + (*start)++; + break; + } + (*start)++; + } + } else if ((*start)[1] == 't' && (*start)[2] == 'i') { + (*start) += offset[index++]; + element->timestamp = *start; + if (*(*start) == '{') { + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->timestampLen = tmp - (*start); + *start = tmp; + } + } else { + while (*(*start)) { + if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { + element->timestampLen = (*start) - element->timestamp; + break; + } + (*start)++; + } + } + } else if ((*start)[1] == 'v') { + (*start) += offset[index++]; + element->cols = *start; + if (*(*start) == '{') { + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->colsLen = tmp - (*start); + *start = tmp; + } + } else { + while (*(*start)) { + if (unlikely(*(*start) == ',' || *(*start) == '}' || (*(*start)) <= 32)) { + element->colsLen = (*start) - element->cols; + break; + } + (*start)++; + } + } + } else if ((*start)[1] == 't' && (*start)[2] == 'a') { + (*start) += offset[index++]; + element->tags = (*start); + char *tmp = smlJsonGetObj((*start)); + if (tmp) { + element->tagsLen = tmp - (*start); + *start = tmp; + } + } + if (*(*start) == '}') { + (*start)++; + break; + } + (*start)++; + } + + if (unlikely(index != 0 && index != OTD_JSON_FIELDS_NUM)) { + uError("elements != %d", OTD_JSON_FIELDS_NUM) return -1; + } + return 0; +} + +static inline int32_t smlParseMetricFromJSON(SSmlHandle *info, cJSON *metric, SSmlLineInfo *elements) { + elements->measureLen = strlen(metric->valuestring); + if (IS_INVALID_TABLE_LEN(elements->measureLen)) { + uError("OTD:0x%" PRIx64 " Metric lenght is 0 or large than 192", info->id); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + elements->measure = metric->valuestring; + return TSDB_CODE_SUCCESS; +} + +const char *jsonName[OTD_JSON_FIELDS_NUM] = {"metric", "timestamp", "value", "tags"}; +static int32_t smlGetJsonElements(cJSON *root, cJSON ***marks) { + for (int i = 0; i < OTD_JSON_FIELDS_NUM; ++i) { + cJSON *child = root->child; + while (child != NULL) { + if (strcasecmp(child->string, jsonName[i]) == 0) { + *marks[i] = child; + break; + } + child = child->next; + } + if (*marks[i] == NULL) { + uError("smlGetJsonElements error, not find mark:%d:%s", i, jsonName[i]); + return -1; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONBool(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "bool") != 0) { + uError("OTD:invalid type(%s) for JSON Bool", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valueint; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlConvertJSONNumber(SSmlKv *pVal, char *typeStr, cJSON *value) { + // tinyint + if (strcasecmp(typeStr, "i8") == 0 || strcasecmp(typeStr, "tinyint") == 0) { + if (!IS_VALID_TINYINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(tinyint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_TINYINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // smallint + if (strcasecmp(typeStr, "i16") == 0 || strcasecmp(typeStr, "smallint") == 0) { + if (!IS_VALID_SMALLINT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(smallint)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_SMALLINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // int + if (strcasecmp(typeStr, "i32") == 0 || strcasecmp(typeStr, "int") == 0) { + if (!IS_VALID_INT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(int)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_INT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->i = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // bigint + if (strcasecmp(typeStr, "i64") == 0 || strcasecmp(typeStr, "bigint") == 0) { + pVal->type = TSDB_DATA_TYPE_BIGINT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + if (value->valuedouble >= (double)INT64_MAX) { + pVal->i = INT64_MAX; + } else if (value->valuedouble <= (double)INT64_MIN) { + pVal->i = INT64_MIN; + } else { + pVal->i = value->valuedouble; + } + return TSDB_CODE_SUCCESS; + } + // float + if (strcasecmp(typeStr, "f32") == 0 || strcasecmp(typeStr, "float") == 0) { + if (!IS_VALID_FLOAT(value->valuedouble)) { + uError("OTD:JSON value(%f) cannot fit in type(float)", value->valuedouble); + return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + } + pVal->type = TSDB_DATA_TYPE_FLOAT; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->f = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + // double + if (strcasecmp(typeStr, "f64") == 0 || strcasecmp(typeStr, "double") == 0) { + pVal->type = TSDB_DATA_TYPE_DOUBLE; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + pVal->d = value->valuedouble; + return TSDB_CODE_SUCCESS; + } + + // if reach here means type is unsupported + uError("OTD:invalid type(%s) for JSON Number", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; +} + +static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { + if (strcasecmp(typeStr, "binary") == 0) { + pVal->type = TSDB_DATA_TYPE_BINARY; + } else if (strcasecmp(typeStr, "nchar") == 0) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + } else { + uError("OTD:invalid type(%s) for JSON String", typeStr); + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + pVal->length = (int16_t)strlen(value->valuestring); + + if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + if (pVal->type == TSDB_DATA_TYPE_NCHAR && + pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + pVal->value = value->valuestring; + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSONObj(cJSON *root, SSmlKv *kv) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t size = cJSON_GetArraySize(root); + + if (size != OTD_JSON_SUB_FIELDS_NUM) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (value == NULL) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (!cJSON_IsString(type)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + + switch (value->type) { + case cJSON_True: + case cJSON_False: { + ret = smlConvertJSONBool(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_Number: { + ret = smlConvertJSONNumber(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + case cJSON_String: { + ret = smlConvertJSONString(kv, type->valuestring, value); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON_TYPE; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseValueFromJSON(cJSON *root, SSmlKv *kv) { + switch (root->type) { + case cJSON_True: + case cJSON_False: { + kv->type = TSDB_DATA_TYPE_BOOL; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->i = root->valueint; + break; + } + case cJSON_Number: { + kv->type = TSDB_DATA_TYPE_DOUBLE; + kv->length = (int16_t)tDataTypes[kv->type].bytes; + kv->d = root->valuedouble; + break; + } + case cJSON_String: { + /* set default JSON type to binary/nchar according to + * user configured parameter tsDefaultJSONStrType + */ + + char *tsDefaultJSONStrType = "binary"; // todo + smlConvertJSONString(kv, tsDefaultJSONStrType, root); + break; + } + case cJSON_Object: { + int32_t ret = smlParseValueFromJSONObj(root, kv); + if (ret != TSDB_CODE_SUCCESS) { + uError("OTD:Failed to parse value from JSON Obj"); + return ret; + } + break; + } + default: + return TSDB_CODE_TSC_INVALID_JSON; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseTagsFromJSON(SSmlHandle *info, cJSON *tags, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if (info->dataFormat) { + if (unlikely(!isSameMeasure)) { + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); + tmp = &sMeta; + } + info->currSTableMeta = (*tmp)->tableMeta; + superKV = (*tmp)->tags; + + if (unlikely(taosArrayGetSize(superKV) == 0)) { + isSuperKVInit = false; + } + taosArrayClear(maxKVs); + } + } else { + taosArrayClear(maxKVs); + } + taosArrayClear(preLineKV); + + int32_t tagNum = cJSON_GetArraySize(tags); + if (unlikely(tagNum == 0)) { + uError("SML:Tag should not be empty"); + return TSDB_CODE_TSC_INVALID_JSON; + } + for (int32_t i = 0; i < tagNum; ++i) { + cJSON *tag = cJSON_GetArrayItem(tags, i); + if (unlikely(tag == NULL)) { + return TSDB_CODE_TSC_INVALID_JSON; + } + // if(unlikely(tag == cMeasure)) continue; + size_t keyLen = strlen(tag->string); + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + uError("OTD:Tag key length is 0 or too large than 64"); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // add kv to SSmlKv + SSmlKv kv = {.key = tag->string, .keyLen = keyLen}; + // value + ret = smlParseValueFromJSON(tag, &kv); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if (info->dataFormat) { + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (isSameMeasure) { + if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + SSmlSTableMeta **tableMeta = + (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + if (unlikely(NULL == tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + if (isSuperKVInit) { + if (unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + } else { + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + } else { + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + + elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); + memcpy(elements->measureTag, elements->measure, elements->measureLen); + memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + elements->measureTagsLen = elements->measureLen + elements->tagsLen; + + SSmlTableInfo **tmp = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); + SSmlTableInfo *tinfo = NULL; + if (unlikely(tmp == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + // *key = *elements; + // if(info->parseJsonByLib){ + // key->tags = taosMemoryMalloc(elements->tagsLen + 1); + // memcpy(key->tags, elements->tags, elements->tagsLen); + // key->tags[elements->tagsLen] = 0; + // } + // tinfo->key = key; + taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, + POINTER_BYTES); + tmp = &tinfo; + } + if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; + + return ret; +} + +static int64_t smlParseTSFromJSONObj(SSmlHandle *info, cJSON *root, int32_t toPrecision) { + int32_t size = cJSON_GetArraySize(root); + if (unlikely(size != OTD_JSON_SUB_FIELDS_NUM)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *value = cJSON_GetObjectItem(root, "value"); + if (unlikely(!cJSON_IsNumber(value))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + cJSON *type = cJSON_GetObjectItem(root, "type"); + if (unlikely(!cJSON_IsString(type))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } + + double timeDouble = value->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (timeDouble == 0) { + return taosGetTimestampNs() / smlFactorNS[toPrecision]; + } + + if (timeDouble < 0) { + return timeDouble; + } + + int64_t tsInt64 = timeDouble; + size_t typeLen = strlen(type->valuestring); + if (typeLen == 1 && (type->valuestring[0] == 's' || type->valuestring[0] == 'S')) { + // seconds + int8_t fromPrecision = TSDB_TIME_PRECISION_SECONDS; + if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + } else if (typeLen == 2 && (type->valuestring[1] == 's' || type->valuestring[1] == 'S')) { + switch (type->valuestring[0]) { + case 'm': + case 'M': + // milliseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MILLI, toPrecision); + break; + case 'u': + case 'U': + // microseconds + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_MICRO, toPrecision); + break; + case 'n': + case 'N': + return convertTimePrecision(tsInt64, TSDB_TIME_PRECISION_NANO, toPrecision); + break; + default: + return -1; + } + } else { + return -1; + } +} + +uint8_t smlGetTimestampLen(int64_t num) { + uint8_t len = 0; + while ((num /= 10) != 0) { + len++; + } + len++; + return len; +} + +static int64_t smlParseTSFromJSON(SSmlHandle *info, cJSON *timestamp) { + // Timestamp must be the first KV to parse + int32_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + if (cJSON_IsNumber(timestamp)) { + // timestamp value 0 indicates current system time + double timeDouble = timestamp->valuedouble; + if (unlikely(smlDoubleToInt64OverFlow(timeDouble))) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is too large", NULL); + return -1; + } + + if (unlikely(timeDouble < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp is negative", NULL); + return timeDouble; + } else if (unlikely(timeDouble == 0)) { + return taosGetTimestampNs() / smlFactorNS[toPrecision]; + } + + uint8_t tsLen = smlGetTimestampLen((int64_t)timeDouble); + + int8_t fromPrecision = smlGetTsTypeByLen(tsLen); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", NULL); + return -1; + } + int64_t tsInt64 = timeDouble; + if (fromPrecision == TSDB_TIME_PRECISION_SECONDS) { + if (smlFactorS[toPrecision] < INT64_MAX / tsInt64) { + return tsInt64 * smlFactorS[toPrecision]; + } + return -1; + } else { + return convertTimePrecision(timeDouble, fromPrecision, toPrecision); + } + } else if (cJSON_IsObject(timestamp)) { + return smlParseTSFromJSONObj(info, timestamp, toPrecision); + } else { + smlBuildInvalidDataMsg(&info->msgBuf, "invalidate json", NULL); + return -1; + } +} + +static int32_t smlParseJSONStringExt(SSmlHandle *info, cJSON *root, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + cJSON *metricJson = NULL; + cJSON *tsJson = NULL; + cJSON *valueJson = NULL; + cJSON *tagsJson = NULL; + + int32_t size = cJSON_GetArraySize(root); + // outmost json fields has to be exactly 4 + if (size != OTD_JSON_FIELDS_NUM) { + uError("OTD:0x%" PRIx64 " Invalid number of JSON fields in data point %d", info->id, size); + return TSDB_CODE_TSC_INVALID_JSON; + } + + cJSON **marks[OTD_JSON_FIELDS_NUM] = {&metricJson, &tsJson, &valueJson, &tagsJson}; + ret = smlGetJsonElements(root, marks); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + // Parse metric + ret = smlParseMetricFromJSON(info, metricJson, elements); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric from JSON payload", info->id); + return ret; + } + + // Parse metric value + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN}; + ret = smlParseValueFromJSON(valueJson, &kv); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse metric value from JSON payload", info->id); + return ret; + } + + // Parse tags + bool needFree = info->dataFormat; + elements->tags = cJSON_PrintUnformatted(tagsJson); + elements->tagsLen = strlen(elements->tags); + if (is_same_child_table_telnet(elements, &info->preLine) != 0) { + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + taosMemoryFree(elements->tags); + elements->tags = NULL; + return ret; + } + } else { + elements->measureTag = info->preLine.measureTag; + } + + if (needFree) { + taosMemoryFree(elements->tags); + elements->tags = NULL; + } + + if (unlikely(info->reRun)) { + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = smlParseTSFromJSON(info, tsJson); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = {.key = TS, + .keyLen = TS_LEN, + .type = TSDB_DATA_TYPE_TIMESTAMP, + .i = ts, + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if (info->dataFormat) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONExt(SSmlHandle *info, char *payload) { + int32_t payloadNum = 0; + int32_t ret = TSDB_CODE_SUCCESS; + + if (unlikely(payload == NULL)) { + uError("SML:0x%" PRIx64 " empty JSON Payload", info->id); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->root = cJSON_Parse(payload); + if (unlikely(info->root == NULL)) { + uError("SML:0x%" PRIx64 " parse json failed:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + // multiple data points must be sent in JSON array + if (cJSON_IsArray(info->root)) { + payloadNum = cJSON_GetArraySize(info->root); + } else if (cJSON_IsObject(info->root)) { + payloadNum = 1; + } else { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 3:%s", info->id, payload); + return TSDB_CODE_TSC_INVALID_JSON; + } + + info->lineNum = payloadNum; + info->dataFormat = true; + if (unlikely(info->lines != NULL)) { + taosMemoryFree(info->lines); + info->lines = NULL; + } + ret = smlClearForRerun(info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + info->parseJsonByLib = true; + cJSON *head = (payloadNum == 1 && cJSON_IsObject(info->root)) ? info->root : info->root->child; + + int cnt = 0; + cJSON *dataPoint = head; + while (dataPoint) { + if (info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONStringExt(info, dataPoint, &element); + } else { + ret = smlParseJSONStringExt(info, dataPoint, info->lines + cnt); + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 2:%s", info->id, payload); + return ret; + } + + if (unlikely(info->reRun)) { + cnt = 0; + dataPoint = head; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + continue; + } + cnt++; + dataPoint = dataPoint->next; + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseJSONString(SSmlHandle *info, char **start, SSmlLineInfo *elements) { + int32_t ret = TSDB_CODE_SUCCESS; + + if (info->offset[0] == 0) { + ret = smlJsonParseObjFirst(start, elements, info->offset); + } else { + ret = smlJsonParseObj(start, elements, info->offset); + } + + if (ret != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(**start == '\0' && elements->measure == NULL)) return TSDB_CODE_SUCCESS; + + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + + if (unlikely(elements->colsLen == 0)) { + uError("SML:colsLen == 0"); + return TSDB_CODE_TSC_INVALID_VALUE; + } else if (unlikely(elements->cols[0] == '{')) { + char tmp = elements->cols[elements->colsLen]; + elements->cols[elements->colsLen] = '\0'; + cJSON *valueJson = cJSON_Parse(elements->cols); + if (unlikely(valueJson == NULL)) { + uError("SML:0x%" PRIx64 " parse json cols failed:%s", info->id, elements->cols); + return TSDB_CODE_TSC_INVALID_JSON; + } + taosArrayPush(info->tagJsonArray, &valueJson); + ret = smlParseValueFromJSONObj(valueJson, &kv); + if (ret != TSDB_CODE_SUCCESS) { + uError("SML:Failed to parse value from JSON Obj:%s", elements->cols); + elements->cols[elements->colsLen] = tmp; + return TSDB_CODE_TSC_INVALID_VALUE; + } + elements->cols[elements->colsLen] = tmp; + } else if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + uError("SML:cols invalidate:%s", elements->cols); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // Parse tags + if (is_same_child_table_telnet(elements, &info->preLine) != 0) { + char tmp = *(elements->tags + elements->tagsLen); + *(elements->tags + elements->tagsLen) = 0; + cJSON *tagsJson = cJSON_Parse(elements->tags); + *(elements->tags + elements->tagsLen) = tmp; + if (unlikely(tagsJson == NULL)) { + uError("SML:0x%" PRIx64 " parse json tag failed:%s", info->id, elements->tags); + return TSDB_CODE_TSC_INVALID_JSON; + } + + taosArrayPush(info->tagJsonArray, &tagsJson); + ret = smlParseTagsFromJSON(info, tagsJson, elements); + if (unlikely(ret)) { + uError("OTD:0x%" PRIx64 " Unable to parse tags from JSON payload", info->id); + return ret; + } + } else { + elements->measureTag = info->preLine.measureTag; + } + + if (unlikely(info->reRun)) { + return TSDB_CODE_SUCCESS; + } + + // Parse timestamp + // notice!!! put ts back to tag to ensure get meta->precision + int64_t ts = 0; + if (unlikely(elements->timestampLen == 0)) { + uError("OTD:0x%" PRIx64 " elements->timestampLen == 0", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } else if (elements->timestamp[0] == '{') { + char tmp = elements->timestamp[elements->timestampLen]; + elements->cols[elements->timestampLen] = '\0'; + cJSON *tsJson = cJSON_Parse(elements->timestamp); + ts = smlParseTSFromJSON(info, tsJson); + if (unlikely(ts < 0)) { + uError("SML:0x%" PRIx64 " Unable to parse timestamp from JSON payload:%s", info->id, elements->timestamp); + elements->timestamp[elements->timestampLen] = tmp; + cJSON_Delete(tsJson); + return TSDB_CODE_INVALID_TIMESTAMP; + } + elements->timestamp[elements->timestampLen] = tmp; + cJSON_Delete(tsJson); + } else { + ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + uError("OTD:0x%" PRIx64 " Unable to parse timestamp from JSON payload", info->id); + return TSDB_CODE_INVALID_TIMESTAMP; + } + } + SSmlKv kvTs = {.key = TS, + .keyLen = TS_LEN, + .type = TSDB_DATA_TYPE_TIMESTAMP, + .i = ts, + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + if (info->dataFormat) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseJSON(SSmlHandle *info, char *payload) { + int32_t payloadNum = 1 << 15; + int32_t ret = TSDB_CODE_SUCCESS; + + uDebug("SML:0x%" PRIx64 "json:%s", info->id, payload); + int cnt = 0; + char *dataPointStart = payload; + while (1) { + if (info->dataFormat) { + SSmlLineInfo element = {0}; + ret = smlParseJSONString(info, &dataPointStart, &element); + if (element.measureTagsLen != 0) taosMemoryFree(element.measureTag); + } else { + if (cnt >= payloadNum) { + payloadNum = payloadNum << 1; + void *tmp = taosMemoryRealloc(info->lines, payloadNum * sizeof(SSmlLineInfo)); + if (tmp != NULL) { + info->lines = (SSmlLineInfo *)tmp; + memset(info->lines + cnt, 0, (payloadNum - cnt) * sizeof(SSmlLineInfo)); + } + } + ret = smlParseJSONString(info, &dataPointStart, info->lines + cnt); + if ((info->lines + cnt)->measure == NULL) break; + } + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("SML:0x%" PRIx64 " Invalid JSON Payload 1:%s", info->id, payload); + return smlParseJSONExt(info, payload); + } + + if (unlikely(info->reRun)) { + cnt = 0; + dataPointStart = payload; + info->lineNum = payloadNum; + ret = smlClearForRerun(info); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + continue; + } + + cnt++; + if (*dataPointStart == '\0') break; + } + info->lineNum = cnt; + + return TSDB_CODE_SUCCESS; +} diff --git a/source/client/src/clientSmlLine.c b/source/client/src/clientSmlLine.c new file mode 100644 index 0000000000000000000000000000000000000000..a2f752edb9f9468a8270f1f8ca094221c718fc6c --- /dev/null +++ b/source/client/src/clientSmlLine.c @@ -0,0 +1,674 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +// comma , +// #define IS_SLASH_COMMA(sql) (*(sql) == COMMA && *((sql)-1) == SLASH) +#define IS_COMMA(sql) (*(sql) == COMMA && *((sql)-1) != SLASH) +// space +// #define IS_SLASH_SPACE(sql) (*(sql) == SPACE && *((sql)-1) == SLASH) +#define IS_SPACE(sql) (*(sql) == SPACE && *((sql)-1) != SLASH) +// equal = +// #define IS_SLASH_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) == SLASH) +#define IS_EQUAL(sql) (*(sql) == EQUAL && *((sql)-1) != SLASH) +// quote " +// #define IS_SLASH_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) == SLASH) +#define IS_QUOTE(sql) (*(sql) == QUOTE && *((sql)-1) != SLASH) +// SLASH +// #define IS_SLASH_SLASH(sql) (*(sql) == SLASH && *((sql)-1) == SLASH) + +#define IS_SLASH_LETTER(sql) \ + (*((sql)-1) == SLASH && (*(sql) == COMMA || *(sql) == SPACE || *(sql) == EQUAL || *(sql) == QUOTE || \ + *(sql) == SLASH)) // (IS_SLASH_COMMA(sql) || IS_SLASH_SPACE(sql) || IS_SLASH_EQUAL(sql) || + // IS_SLASH_QUOTE(sql) || IS_SLASH_SLASH(sql)) + +#define MOVE_FORWARD_ONE(sql, len) (memmove((void *)((sql)-1), (sql), len)) + +#define PROCESS_SLASH(key, keyLen) \ + for (int i = 1; i < keyLen; ++i) { \ + if (IS_SLASH_LETTER(key + i)) { \ + MOVE_FORWARD_ONE(key + i, keyLen - i); \ + i--; \ + keyLen--; \ + } \ + } + +#define BINARY_ADD_LEN 2 // "binary" 2 means " " +#define NCHAR_ADD_LEN 3 // L"nchar" 3 means L" " + +uint8_t smlPrecisionConvert[7] = {TSDB_TIME_PRECISION_NANO, TSDB_TIME_PRECISION_HOURS, TSDB_TIME_PRECISION_MINUTES, + TSDB_TIME_PRECISION_SECONDS, TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, + TSDB_TIME_PRECISION_NANO}; + +static int64_t smlParseInfluxTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(len == 0 || (len == 1 && data[0] == '0'))) { + return taosGetTimestampNs() / smlFactorNS[toPrecision]; + } + + uint8_t fromPrecision = smlPrecisionConvert[info->precision]; + + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +int32_t smlParseValue(SSmlKv *pVal, SSmlMsgBuf *msg) { + if (pVal->value[0] == '"') { // binary + if (pVal->length >= 2 && pVal->value[pVal->length - 1] == '"') { + pVal->type = TSDB_DATA_TYPE_BINARY; + pVal->length -= BINARY_ADD_LEN; + if (pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (BINARY_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'l' || pVal->value[0] == 'L') { // nchar + if (pVal->value[1] == '"' && pVal->value[pVal->length - 1] == '"' && pVal->length >= 3) { + pVal->type = TSDB_DATA_TYPE_NCHAR; + pVal->length -= NCHAR_ADD_LEN; + if (pVal->length > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + pVal->value += (NCHAR_ADD_LEN - 1); + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 't' || pVal->value[0] == 'T') { + if (pVal->length == 1 || + (pVal->length == 4 && (pVal->value[1] == 'r' || pVal->value[1] == 'R') && + (pVal->value[2] == 'u' || pVal->value[2] == 'U') && (pVal->value[3] == 'e' || pVal->value[3] == 'E'))) { + pVal->i = TSDB_TRUE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (pVal->value[0] == 'f' || pVal->value[0] == 'F') { + if (pVal->length == 1 || + (pVal->length == 5 && (pVal->value[1] == 'a' || pVal->value[1] == 'A') && + (pVal->value[2] == 'l' || pVal->value[2] == 'L') && (pVal->value[3] == 's' || pVal->value[3] == 'S') && + (pVal->value[4] == 'e' || pVal->value[4] == 'E'))) { + pVal->i = TSDB_FALSE; + pVal->type = TSDB_DATA_TYPE_BOOL; + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + return TSDB_CODE_TSC_INVALID_VALUE; + } + + // number + if (smlParseNumber(pVal, msg)) { + pVal->length = (int16_t)tDataTypes[pVal->type].bytes; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_TSC_INVALID_VALUE; +} + +static int32_t smlParseTagKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, + bool isSameCTable) { + if (isSameCTable) { + return TSDB_CODE_SUCCESS; + } + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if (info->dataFormat) { + if (unlikely(!isSameMeasure)) { + SSmlSTableMeta **tmp = + (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + tmp = &sMeta; + } + info->currSTableMeta = (*tmp)->tableMeta; + superKV = (*tmp)->tags; + + if (unlikely(taosArrayGetSize(superKV) == 0)) { + isSuperKVInit = false; + } + taosArrayClear(maxKVs); + } + } else { + taosArrayClear(maxKVs); + } + taosArrayClear(preLineKV); + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if (!hasSlash) { + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if (unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + if (!hasSlash) { + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + + if (unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + if (info->dataFormat) { + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (isSameMeasure) { + if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + SSmlSTableMeta **tableMeta = + (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); + if (unlikely(NULL == tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + if (isSuperKVInit) { + if (unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + } else { + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + } else { + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + + cnt++; + if (IS_SPACE(*sql)) { + break; + } + (*sql)++; + } + + void *oneTable = taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); + if ((oneTable != NULL)) { + return TSDB_CODE_SUCCESS; + } + + SSmlTableInfo *tinfo = smlBuildTableInfo(1, currElement->measure, currElement->measureLen); + if (unlikely(!tinfo)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + taosHashPut(info->childTables, currElement->measure, currElement->measureTagsLen, &tinfo, POINTER_BYTES); + + return TSDB_CODE_SUCCESS; +} + +static int32_t smlParseColKv(SSmlHandle *info, char **sql, char *sqlEnd, SSmlLineInfo *currElement, bool isSameMeasure, + bool isSameCTable) { + int cnt = 0; + SArray *preLineKV = info->preLineColKV; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if (info->dataFormat) { + if (unlikely(!isSameCTable)) { + SSmlTableInfo **oneTable = + (SSmlTableInfo **)taosHashGet(info->childTables, currElement->measure, currElement->measureTagsLen); + if (unlikely(oneTable == NULL)) { + smlBuildInvalidDataMsg(&info->msgBuf, "child table should inside", currElement->measure); + return TSDB_CODE_SML_INVALID_DATA; + } + info->currTableDataCtx = (*oneTable)->tableDataCtx; + } + + if (unlikely(!isSameMeasure)) { + SSmlSTableMeta **tmp = + (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + STableMeta *pTableMeta = smlGetMeta(info, currElement->measure, currElement->measureLen); + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, currElement->measure, currElement->measureLen, &sMeta, POINTER_BYTES); + tmp = &sMeta; + } + info->currSTableMeta = (*tmp)->tableMeta; + superKV = (*tmp)->cols; + if (unlikely(taosArrayGetSize(superKV) == 0)) { + isSuperKVInit = false; + } + taosArrayClear(preLineKV); + } + } + + while (*sql < sqlEnd) { + if (unlikely(IS_SPACE(*sql))) { + break; + } + + bool hasSlash = false; + // parse key + const char *key = *sql; + size_t keyLen = 0; + while (*sql < sqlEnd) { + if (unlikely(IS_COMMA(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(IS_EQUAL(*sql))) { + keyLen = *sql - key; + (*sql)++; + break; + } + if (!hasSlash) { + hasSlash = (*(*sql) == SLASH); + } + (*sql)++; + } + if (unlikely(hasSlash)) { + PROCESS_SLASH(key, keyLen) + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + + // parse value + const char *value = *sql; + size_t valueLen = 0; + hasSlash = false; + bool isInQuote = false; + while (*sql < sqlEnd) { + // parse value + if (unlikely(IS_QUOTE(*sql))) { + isInQuote = !isInQuote; + (*sql)++; + continue; + } + if (!isInQuote) { + if (unlikely(IS_SPACE(*sql) || IS_COMMA(*sql))) { + break; + } else if (unlikely(IS_EQUAL(*sql))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", *sql); + return TSDB_CODE_SML_INVALID_DATA; + } + } + if (!hasSlash) { + hasSlash = (*(*sql) == SLASH); + } + + (*sql)++; + } + valueLen = *sql - value; + + if (unlikely(isInQuote)) { + smlBuildInvalidDataMsg(&info->msgBuf, "only one quote", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(hasSlash)) { + PROCESS_SLASH(value, valueLen) + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .value = value, .length = valueLen}; + int32_t ret = smlParseValue(&kv, &info->msgBuf); + if (ret != TSDB_CODE_SUCCESS) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlParseValue error", value); + return ret; + } + + if (info->dataFormat) { + // cnt begin 0, add ts so + 2 + if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + // bind data + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, cnt + 1); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + uError("smlBuildCol error, retry"); + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (isSameMeasure) { + if (cnt >= taosArrayGetSize(preLineKV)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(preLineKV, cnt); + if (kv.type != maxKV->type) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (unlikely(IS_VAR_DATA_TYPE(kv.type) && kv.length > maxKV->length)) { + maxKV->length = kv.length; + SSmlSTableMeta **tableMeta = + (SSmlSTableMeta **)taosHashGet(info->superTables, currElement->measure, currElement->measureLen); + if (unlikely(NULL == tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->cols, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + if (isSuperKVInit) { + if (unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if (unlikely(kv.type != maxKV->type)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (IS_VAR_DATA_TYPE(kv.type)) { + if (kv.length > maxKV->length) { + maxKV->length = kv.length; + } else { + kv.length = maxKV->length; + } + info->needModifySchema = true; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + taosArrayPush(superKV, &kv); + } + taosArrayPush(preLineKV, &kv); + } + } else { + if (currElement->colArray == NULL) { + currElement->colArray = taosArrayInit_s(sizeof(SSmlKv), 1); + } + taosArrayPush(currElement->colArray, &kv); // reserve for timestamp + } + + cnt++; + if (IS_SPACE(*sql)) { + break; + } + (*sql)++; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t smlParseInfluxString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == COMMA)) return TSDB_CODE_SML_INVALID_DATA; + elements->measure = sql; + + // parse measure + while (sql < sqlEnd) { + if (unlikely((sql != elements->measure) && IS_SLASH_LETTER(sql))) { + MOVE_FORWARD_ONE(sql, sqlEnd - sql); + sqlEnd--; + continue; + } + if (unlikely(IS_COMMA(sql))) { + break; + } + + if (unlikely(IS_SPACE(sql))) { + break; + } + sql++; + } + elements->measureLen = sql - elements->measure; + if (unlikely(IS_INVALID_TABLE_LEN(elements->measureLen))) { + smlBuildInvalidDataMsg(&info->msgBuf, "measure is empty or too large than 192", NULL); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // to get measureTagsLen before + const char *tmp = sql; + while (tmp < sqlEnd) { + if (unlikely(IS_SPACE(tmp))) { + break; + } + tmp++; + } + elements->measureTagsLen = tmp - elements->measure; + + bool isSameCTable = false; + bool isSameMeasure = false; + if (IS_SAME_CHILD_TABLE) { + isSameCTable = true; + isSameMeasure = true; + } else if (info->dataFormat) { + isSameMeasure = IS_SAME_SUPER_TABLE; + } + // parse tag + if (*sql == COMMA) sql++; + elements->tags = sql; + + int ret = smlParseTagKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + if (unlikely(info->reRun)) { + return TSDB_CODE_SUCCESS; + } + + sql = elements->measure + elements->measureTagsLen; + elements->tagsLen = sql - elements->tags; + + // parse cols + JUMP_SPACE(sql, sqlEnd) + elements->cols = sql; + + ret = smlParseColKv(info, &sql, sqlEnd, elements, isSameMeasure, isSameCTable); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if (unlikely(info->reRun)) { + return TSDB_CODE_SUCCESS; + } + + elements->colsLen = sql - elements->cols; + if (unlikely(elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "cols is empty", NULL); + return TSDB_CODE_SML_INVALID_DATA; + } + + // parse timestamp + JUMP_SPACE(sql, sqlEnd) + elements->timestamp = sql; + while (sql < sqlEnd) { + if (unlikely(isspace(*sql))) { + break; + } + sql++; + } + elements->timestampLen = sql - elements->timestamp; + + int64_t ts = smlParseInfluxTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts <= 0)) { + uError("SML:0x%" PRIx64 " smlParseTS error:%" PRId64, info->id, ts); + return TSDB_CODE_INVALID_TIMESTAMP; + } + // add ts to + SSmlKv kv = {.key = TS, + .keyLen = TS_LEN, + .type = TSDB_DATA_TYPE_TIMESTAMP, + .i = ts, + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + if (info->dataFormat) { + smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 0); + smlBuildRow(info->currTableDataCtx); + clearColValArray(info->currTableDataCtx->pValues); + } else { + taosArraySet(elements->colArray, 0, &kv); + } + info->preLine = *elements; + + return ret; +} diff --git a/source/client/src/clientSmlTelnet.c b/source/client/src/clientSmlTelnet.c new file mode 100644 index 0000000000000000000000000000000000000000..ab071305fa7777de74546063a1ccfca56b193bc7 --- /dev/null +++ b/source/client/src/clientSmlTelnet.c @@ -0,0 +1,364 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include + +#include "clientSml.h" + +int32_t is_same_child_table_telnet(const void *a, const void *b) { + SSmlLineInfo *t1 = (SSmlLineInfo *)a; + SSmlLineInfo *t2 = (SSmlLineInfo *)b; + // uError("is_same_child_table_telnet len:%d,%d %s,%s @@@ len:%d,%d %s,%s", t1->measureLen, t2->measureLen, + // t1->measure, t2->measure, t1->tagsLen, t2->tagsLen, t1->tags, t2->tags); + if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL) + return 1; + return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) && + ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0)) + ? 0 + : 1; +} + +int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) { + uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO; + + if (unlikely(!data)) { + smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL); + return -1; + } + if (unlikely(len == 1 && data[0] == '0')) { + return taosGetTimestampNs() / smlFactorNS[toPrecision]; + } + int8_t fromPrecision = smlGetTsTypeByLen(len); + if (unlikely(fromPrecision == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, + "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data); + return -1; + } + int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision); + if (unlikely(ts == -1)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data); + return -1; + } + return ts; +} + +static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) { + while (*sql < sqlEnd) { + if (unlikely((**sql != SPACE && !(*data)))) { + *data = *sql; + } else if (unlikely(**sql == SPACE && *data)) { + *len = *sql - *data; + break; + } + (*sql)++; + } +} + +static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements, SSmlMsgBuf *msg) { + if (is_same_child_table_telnet(elements, &info->preLine) == 0) { + elements->measureTag = info->preLine.measureTag; + return TSDB_CODE_SUCCESS; + } + + bool isSameMeasure = IS_SAME_SUPER_TABLE; + + int cnt = 0; + SArray *preLineKV = info->preLineTagKV; + SArray *maxKVs = info->maxTagKVs; + bool isSuperKVInit = true; + SArray *superKV = NULL; + if (info->dataFormat) { + if (!isSameMeasure) { + SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + SSmlSTableMeta *sMeta = NULL; + if (unlikely(tmp == NULL)) { + STableMeta *pTableMeta = smlGetMeta(info, elements->measure, elements->measureLen); + if (pTableMeta == NULL) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + sMeta = smlBuildSTableMeta(info->dataFormat); + sMeta->tableMeta = pTableMeta; + taosHashPut(info->superTables, elements->measure, elements->measureLen, &sMeta, POINTER_BYTES); + tmp = &sMeta; + } + info->currSTableMeta = (*tmp)->tableMeta; + superKV = (*tmp)->tags; + + if (unlikely(taosArrayGetSize(superKV) == 0)) { + isSuperKVInit = false; + } + taosArrayClear(maxKVs); + } + } else { + taosArrayClear(maxKVs); + } + + taosArrayClear(preLineKV); + const char *sql = data; + while (sql < sqlEnd) { + JUMP_SPACE(sql, sqlEnd) + if (unlikely(*sql == '\0')) break; + + const char *key = sql; + size_t keyLen = 0; + + // parse key + while (sql < sqlEnd) { + if (unlikely(*sql == SPACE)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + if (unlikely(*sql == EQUAL)) { + keyLen = sql - key; + sql++; + break; + } + sql++; + } + + if (unlikely(IS_INVALID_COL_LEN(keyLen))) { + smlBuildInvalidDataMsg(msg, "invalid key or key is too long than 64", key); + return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH; + } + // if (smlCheckDuplicateKey(key, keyLen, dumplicateKey)) { + // smlBuildInvalidDataMsg(msg, "dumplicate key", key); + // return TSDB_CODE_TSC_DUP_NAMES; + // } + + // parse value + const char *value = sql; + size_t valueLen = 0; + while (sql < sqlEnd) { + // parse value + if (unlikely(*sql == SPACE)) { + break; + } + if (unlikely(*sql == EQUAL)) { + smlBuildInvalidDataMsg(msg, "invalid data", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + sql++; + } + valueLen = sql - value; + + if (unlikely(valueLen == 0)) { + smlBuildInvalidDataMsg(msg, "invalid value", value); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (unlikely(valueLen > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) { + return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; + } + + SSmlKv kv = {.key = key, .keyLen = keyLen, .type = TSDB_DATA_TYPE_NCHAR, .value = value, .length = valueLen}; + + if (info->dataFormat) { + if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + + if (isSameMeasure) { + if (unlikely(cnt >= taosArrayGetSize(maxKVs))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(maxKVs, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + SSmlSTableMeta **tableMeta = + (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); + if (unlikely(NULL == tableMeta)) { + uError("SML:0x%" PRIx64 " NULL == tableMeta", info->id); + return TSDB_CODE_SML_INTERNAL_ERROR; + } + + SSmlKv *oldKV = (SSmlKv *)taosArrayGet((*tableMeta)->tags, cnt); + oldKV->length = kv.length; + info->needModifySchema = true; + } + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + if (isSuperKVInit) { + if (unlikely(cnt >= taosArrayGetSize(superKV))) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + SSmlKv *maxKV = (SSmlKv *)taosArrayGet(superKV, cnt); + if (unlikely(kv.length > maxKV->length)) { + maxKV->length = kv.length; + } else { + kv.length = maxKV->length; + } + info->needModifySchema = true; + + if (unlikely(!IS_SAME_KEY)) { + info->dataFormat = false; + info->reRun = true; + return TSDB_CODE_SUCCESS; + } + } else { + taosArrayPush(superKV, &kv); + } + taosArrayPush(maxKVs, &kv); + } + } else { + taosArrayPush(maxKVs, &kv); + } + taosArrayPush(preLineKV, &kv); + cnt++; + } + + elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); + memcpy(elements->measureTag, elements->measure, elements->measureLen); + memcpy(elements->measureTag + elements->measureLen, elements->tags, elements->tagsLen); + elements->measureTagsLen = elements->measureLen + elements->tagsLen; + + SSmlTableInfo **tmp = + (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen); + SSmlTableInfo *tinfo = NULL; + if (unlikely(tmp == NULL)) { + tinfo = smlBuildTableInfo(1, elements->measure, elements->measureLen); + if (!tinfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tinfo->tags = taosArrayDup(preLineKV, NULL); + + smlSetCTableName(tinfo); + tinfo->uid = info->uid++; + if (info->dataFormat) { + info->currSTableMeta->uid = tinfo->uid; + tinfo->tableDataCtx = smlInitTableDataCtx(info->pQuery, info->currSTableMeta); + if (tinfo->tableDataCtx == NULL) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlInitTableDataCtx error", NULL); + smlDestroyTableInfo(info, tinfo); + return TSDB_CODE_SML_INVALID_DATA; + } + } + + // SSmlLineInfo *key = (SSmlLineInfo *)taosMemoryMalloc(sizeof(SSmlLineInfo)); + // *key = *elements; + // tinfo->key = key; + taosHashPut(info->childTables, elements->measureTag, elements->measureLen + elements->tagsLen, &tinfo, + POINTER_BYTES); + tmp = &tinfo; + } + if (info->dataFormat) info->currTableDataCtx = (*tmp)->tableDataCtx; + + return TSDB_CODE_SUCCESS; +} + +// format: =[ =] +int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) { + if (!sql) return TSDB_CODE_SML_INVALID_DATA; + + // parse metric + smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen); + if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql); + return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + } + + // parse timestamp + smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen); + if (unlikely(!elements->timestamp || elements->timestampLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_SML_INVALID_DATA; + } + + bool needConverTime = false; // get TS before parse tag(get meta), so need conver time + if (info->dataFormat && info->currSTableMeta == NULL) { + needConverTime = true; + } + int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen); + if (unlikely(ts < 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", sql); + return TSDB_CODE_INVALID_TIMESTAMP; + } + SSmlKv kvTs = {.key = TS, + .keyLen = TS_LEN, + .type = TSDB_DATA_TYPE_TIMESTAMP, + .i = ts, + .length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes}; + + // parse value + smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen); + if (unlikely(!elements->cols || elements->colsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen}; + if (smlParseValue(&kv, &info->msgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + JUMP_SPACE(sql, sqlEnd) + + elements->tags = sql; + elements->tagsLen = sqlEnd - sql; + if (unlikely(!elements->tags || elements->tagsLen == 0)) { + smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", sql); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int ret = smlParseTelnetTags(info, sql, sqlEnd, elements, &info->msgBuf); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + return ret; + } + + if (unlikely(info->reRun)) { + return TSDB_CODE_SUCCESS; + } + + if (info->dataFormat) { + if (needConverTime) { + kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision); + } + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kvTs, 0); + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, &kv, 1); + } + if (ret == TSDB_CODE_SUCCESS) { + ret = smlBuildRow(info->currTableDataCtx); + } + clearColValArray(info->currTableDataCtx->pValues); + if (unlikely(ret != TSDB_CODE_SUCCESS)) { + smlBuildInvalidDataMsg(&info->msgBuf, "smlBuildCol error", NULL); + return ret; + } + } else { + if (elements->colArray == NULL) { + elements->colArray = taosArrayInit(16, sizeof(SSmlKv)); + } + taosArrayPush(elements->colArray, &kvTs); + taosArrayPush(elements->colArray, &kv); + } + info->preLine = *elements; + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 5646f58ee14885b910ec4781bd5c6f0a8620ff47..3ed157efef1fc84b58cfe4d566965123108047cd 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -172,12 +172,11 @@ int32_t stmtUpdateBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, return TSDB_CODE_SUCCESS; } -int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash, bool autoCreateTbl) { +int32_t stmtUpdateExecInfo(TAOS_STMT* stmt, SHashObj* pVgHash, SHashObj* pBlockHash) { STscStmt* pStmt = (STscStmt*)stmt; pStmt->sql.pVgHash = pVgHash; pStmt->exec.pBlockHash = pBlockHash; - pStmt->exec.autoCreateTbl = autoCreateTbl; return TSDB_CODE_SUCCESS; } @@ -187,7 +186,7 @@ int32_t stmtUpdateInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags, SNam STscStmt* pStmt = (STscStmt*)stmt; STMT_ERR_RET(stmtUpdateBindInfo(stmt, pTableMeta, tags, tbName, sTableName, autoCreateTbl)); - STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash, autoCreateTbl)); + STMT_ERR_RET(stmtUpdateExecInfo(stmt, pVgHash, pBlockHash)); pStmt->sql.autoCreateTbl = autoCreateTbl; @@ -215,16 +214,16 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pSrc = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (!pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } - STableDataBlocks* pDst = NULL; + STableDataCxt* pDst = NULL; - STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc)); + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); SStmtTableCache cache = { - .pDataBlock = pDst, + .pDataCtx = pDst, .boundTags = pStmt->bInfo.boundTags, }; @@ -242,6 +241,8 @@ int32_t stmtCacheBlock(STscStmt* pStmt) { } int32_t stmtParseSql(STscStmt* pStmt) { + pStmt->exec.pCurrBlock = NULL; + SStmtCallback stmtCb = { .pStmt = pStmt, .getTbNameFn = stmtGetTbName, @@ -274,7 +275,7 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { pStmt->bInfo.tbName[0] = 0; pStmt->bInfo.tbFName[0] = 0; if (!pStmt->bInfo.tagsCached) { - destroyBoundColumnInfo(pStmt->bInfo.boundTags); + qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } memset(pStmt->bInfo.stbFName, 0, TSDB_TABLE_FNAME_LEN); @@ -290,12 +291,13 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { size_t keyLen = 0; void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlocks = *(STableDataBlocks**)pIter; - char* key = taosHashGetKey(pIter, &keyLen); - STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); + STableDataCxt* pBlocks = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); + STableMeta* pMeta = qGetTableMetaInDataBlock(pBlocks); - if (keepTable && (strlen(pStmt->bInfo.tbFName) == keyLen) && strncmp(pStmt->bInfo.tbFName, key, keyLen) == 0) { - STMT_ERR_RET(qResetStmtDataBlock(pBlocks, true)); + if (keepTable && pBlocks == pStmt->exec.pCurrBlock) { + TSWAP(pBlocks->pData, pStmt->exec.pCurrTbData); + STMT_ERR_RET(qResetStmtDataBlock(pBlocks, false)); pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); continue; @@ -307,8 +309,6 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { pIter = taosHashIterate(pStmt->exec.pBlockHash, pIter); } - pStmt->exec.autoCreateTbl = false; - if (keepTable) { return TSDB_CODE_SUCCESS; } @@ -316,6 +316,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + STMT_ERR_RET(stmtCleanBindInfo(pStmt)); return TSDB_CODE_SUCCESS; @@ -334,8 +337,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { while (pIter) { SStmtTableCache* pCache = (SStmtTableCache*)pIter; - qDestroyStmtDataBlock(pCache->pDataBlock); - destroyBoundColumnInfo(pCache->boundTags); + qDestroyStmtDataBlock(pCache->pDataCtx); + qDestroyBoundColInfo(pCache->boundTags); taosMemoryFreeClear(pCache->boundTags); pIter = taosHashIterate(pStmt->sql.pTableCache, pIter); @@ -351,7 +354,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } -int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STableDataBlocks** newBlock, uint64_t uid) { +int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, + uint64_t suid) { SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -363,7 +367,9 @@ int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataBlocks* pDataBlock, STab STMT_ERR_RET( taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo))); - STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, vgInfo.vgId)); + STMT_ERR_RET(qRebuildStmtDataBlock(newBlock, pDataBlock, uid, suid, vgInfo.vgId, pStmt->sql.autoCreateTbl)); + + STMT_DLOG("tableDataCxt rebuilt, uid:%" PRId64 ", vgId:%d", uid, vgInfo.vgId); return TSDB_CODE_SUCCESS; } @@ -372,12 +378,13 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.needParse = true; pStmt->bInfo.inExecCache = false; - STableDataBlocks* pBlockInExec = - taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (pBlockInExec) { + STableDataCxt** pCxtInExec = taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (pCxtInExec) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = true; + pStmt->exec.pCurrBlock = *pCxtInExec; + if (pStmt->sql.autoCreateTbl) { tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -386,7 +393,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (NULL == pStmt->sql.pTableCache || taosHashGetSize(pStmt->sql.pTableCache) <= 0) { if (pStmt->bInfo.inExecCache) { - ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1); + if (ASSERT(taosHashGetSize(pStmt->exec.pBlockHash) == 1)) { + tscError("stmtGetFromCache error"); + return TSDB_CODE_TSC_STMT_CACHE_ERROR; + } pStmt->bInfo.needParse = false; tscDebug("reuse stmt block for tb %s in execBlock", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -404,18 +414,18 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { pStmt->bInfo.needParse = false; - pStmt->exec.autoCreateTbl = true; - pStmt->bInfo.tbUid = 0; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, 0)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, 0, pStmt->bInfo.tbSuid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("reuse stmt block for tb %s in sqlBlock, suid:0x%" PRIx64, pStmt->bInfo.tbFName, pStmt->bInfo.tbSuid); return TSDB_CODE_SUCCESS; @@ -486,14 +496,16 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.tagsCached = true; - STableDataBlocks* pNewBlock = NULL; - STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataBlock, &pNewBlock, uid)); + STableDataCxt* pNewBlock = NULL; + STMT_ERR_RET(stmtRebuildDataBlock(pStmt, pCache->pDataCtx, &pNewBlock, uid, suid)); if (taosHashPut(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName), &pNewBlock, POINTER_BYTES)) { STMT_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pStmt->exec.pCurrBlock = pNewBlock; + tscDebug("tb %s in sqlBlock list, set to current", pStmt->bInfo.tbFName); return TSDB_CODE_SUCCESS; @@ -611,8 +623,8 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -623,8 +635,6 @@ int stmtSetTbTags(TAOS_STMT* stmt, TAOS_MULTI_BIND* tags) { pStmt->bInfo.sname.tname, tags, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); - pStmt->exec.autoCreateTbl = true; - return TSDB_CODE_SUCCESS; } @@ -634,8 +644,8 @@ int stmtFetchTagFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -652,8 +662,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -726,11 +736,18 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { return TSDB_CODE_SUCCESS; } - STableDataBlocks** pDataBlock = - (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); - if (NULL == pDataBlock) { - tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); - STMT_ERR_RET(TSDB_CODE_APP_ERROR); + STableDataCxt** pDataBlock = NULL; + + if (pStmt->exec.pCurrBlock) { + pDataBlock = &pStmt->exec.pCurrBlock; + } else { + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + if (NULL == pDataBlock) { + tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); + STMT_ERR_RET(TSDB_CODE_TSC_STMT_CACHE_ERROR); + } + pStmt->exec.pCurrBlock = *pDataBlock; } if (colIdx < 0) { @@ -773,13 +790,13 @@ int stmtAddBatch(TAOS_STMT* stmt) { int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { tscDebug("stmt start to update tbUid, blockNum: %d", pRsp->nBlocks); - int32_t code = 0; - int32_t finalCode = 0; - size_t keyLen = 0; - STableDataBlocks** pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); + int32_t code = 0; + int32_t finalCode = 0; + size_t keyLen = 0; + void* pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); while (pIter) { - STableDataBlocks* pBlock = *pIter; - char* key = taosHashGetKey(pIter, &keyLen); + STableDataCxt* pBlock = *(STableDataCxt**)pIter; + char* key = taosHashGetKey(pIter, &keyLen); STableMeta* pMeta = qGetTableMetaInDataBlock(pBlock); if (pMeta->uid) { @@ -858,7 +875,6 @@ int stmtExec(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; int32_t code = 0; SSubmitRsp* pRsp = NULL; - bool autoCreateTbl = pStmt->exec.autoCreateTbl; STMT_DLOG_E("start to exec"); @@ -867,8 +883,13 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { + tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pStmt->exec.pCurrTbData); + + STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); + STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); - launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, (autoCreateTbl ? (void**)&pRsp : NULL)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -891,15 +912,6 @@ _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - if (TSDB_CODE_SUCCESS == code && autoCreateTbl) { - if (NULL == pRsp) { - tscError("no submit resp got for auto create table"); - code = TSDB_CODE_APP_ERROR; - } else { - code = stmtUpdateTableUid(pStmt, pRsp); - } - } - tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index df4b2204b4d1f36ff8e0fc6efd5854e82d840d73..d913b32642578379f7a1e39cfc31d5e528e6e6f1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -32,14 +32,14 @@ sem_post(x) #endif -struct SMqMgmt{ +struct SMqMgmt { int8_t inited; tmr_h timer; int32_t rsetId; }; -static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once -volatile int32_t tmqInitRes = 0; // initialize rsp code +static TdThreadOnce tmqInit = PTHREAD_ONCE_INIT; // initialize only once +volatile int32_t tmqInitRes = 0; // initialize rsp code static struct SMqMgmt tmqMgmt = {0}; typedef struct { @@ -76,7 +76,7 @@ struct tmq_conf_t { }; struct tmq_t { - int64_t refId; + int64_t refId; // conf char groupId[TSDB_CGROUP_LEN]; char clientId[256]; @@ -330,15 +330,15 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcmp(key, "td.connect.ip") == 0) { - conf->ip = strdup(value); + conf->ip = taosStrdup(value); return TMQ_CONF_OK; } if (strcmp(key, "td.connect.user") == 0) { - conf->user = strdup(value); + conf->user = taosStrdup(value); return TMQ_CONF_OK; } if (strcmp(key, "td.connect.pass") == 0) { - conf->pass = strdup(value); + conf->pass = taosStrdup(value); return TMQ_CONF_OK; } if (strcmp(key, "td.connect.port") == 0) { @@ -346,7 +346,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value return TMQ_CONF_OK; } if (strcmp(key, "td.connect.db") == 0) { - /*conf->db = strdup(value);*/ + /*conf->db = taosStrdup(value);*/ return TMQ_CONF_OK; } @@ -361,7 +361,7 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* list, const char* src) { SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; - char* topic = strdup(src); + char* topic = taosStrdup(src); if (topic[0] != '`') { strtolower(topic, src); } @@ -525,8 +525,8 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT }; SEp* pEp = &pVg->epSet.eps[pVg->epSet.inUse]; - tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64" prev:%"PRId64", ep:%s:%d", tmq->consumerId, pOffset->subKey, - pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, pEp->port); + tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d offset:%" PRId64 " prev:%" PRId64 ", ep:%s:%d", tmq->consumerId, + pOffset->subKey, pVg->vgId, pOffset->val.version, pVg->committedOffset.version, pEp->fqdn, pEp->port); // TODO: put into cb, the commit offset should be move to the callback function pVg->committedOffset = pVg->currentOffset; @@ -550,7 +550,6 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { char* topic; int32_t vgId; - if (TD_RES_TMQ(msg)) { SMqRspObj* pRspObj = (SMqRspObj*)msg; topic = pRspObj->topic; @@ -654,7 +653,7 @@ static int32_t tmqCommitConsumerImpl(tmq_t* tmq, int8_t automatic, int8_t async, pParamSet->waitingRspNum = 1; int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%"PRIx64" start to commit offset for %d topics", tmq->consumerId, numOfTopics); + tscDebug("consumer:0x%" PRIx64 " start to commit offset for %d topics", tmq->consumerId, numOfTopics); for (int32_t i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); @@ -813,7 +812,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { return TSDB_CODE_SUCCESS; } - tscDebug("consumer:0x%"PRIx64" handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems); + tscDebug("consumer:0x%" PRIx64 " handle delayed %d tasks before poll data", pTmq->consumerId, qall->numOfItems); int8_t* pTaskType = NULL; taosGetQitem(qall, (void**)&pTaskType); @@ -824,7 +823,7 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; - tscDebug("consumer:0x%"PRIx64" retrieve ep from mnode in 1s", pTmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " retrieve ep from mnode in 1s", pTmq->consumerId); taosTmrReset(tmqAssignAskEpTask, 1000, pRefId, tmqMgmt.timer, &pTmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmqCommitInner(pTmq, NULL, 1, 1, pTmq->commitCb, pTmq->commitCbUserParam); @@ -832,12 +831,10 @@ int32_t tmqHandleAllDelayedTask(tmq_t* pTmq) { int64_t* pRefId = taosMemoryMalloc(sizeof(int64_t)); *pRefId = pTmq->refId; - tscDebug("consumer:0x%"PRIx64" commit to vnode(s) in %.2fs", pTmq->consumerId, pTmq->autoCommitInterval/1000.0); + tscDebug("consumer:0x%" PRIx64 " commit to vnode(s) in %.2fs", pTmq->consumerId, + pTmq->autoCommitInterval / 1000.0); taosTmrReset(tmqAssignDelayedCommitTask, pTmq->autoCommitInterval, pRefId, tmqMgmt.timer, &pTmq->commitTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) { - // do nothing - } else { - ASSERT(0); } taosFreeQitem(pTaskType); @@ -998,7 +995,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->qall = taosAllocateQall(); pTmq->delayedTask = taosOpenQueue(); - if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || conf->groupId[0] == 0) { + if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || + conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tscError("consumer:0x%" PRIx64 " setup failed since %s, consumer group %s", pTmq->consumerId, terrstr(), pTmq->groupId); @@ -1077,7 +1075,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { SCMSubscribeReq req = {0}; int32_t code = 0; - tscDebug("consumer:0x%"PRIx64" subscribe %d topics", tmq->consumerId, sz); + tscDebug("consumer:0x%" PRIx64 " subscribe %d topics", tmq->consumerId, sz); req.consumerId = tmq->consumerId; tstrncpy(req.clientId, tmq->clientId, 256); @@ -1100,7 +1098,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { } tNameExtractFullName(&name, topicFName); - tscDebug("consumer:0x%"PRIx64" subscribe topic:%s", tmq->consumerId, topicFName); + tscDebug("consumer:0x%" PRIx64 " subscribe topic:%s", tmq->consumerId, topicFName); taosArrayPush(req.topicNames, &topicFName); } @@ -1167,7 +1165,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - tscDebug("consumer:0x%"PRIx64", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); + tscDebug("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry:%d in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } @@ -1295,8 +1293,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->taosxRsp, pMsg->pData, sizeof(SMqRspHead)); - } else { - ASSERT(0); } taosMemoryFree(pMsg->pData); @@ -1322,8 +1318,8 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics); - char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; - tscDebug("consumer:0x%" PRIx64" update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", + char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; + tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); @@ -1436,8 +1432,8 @@ int32_t tmqAskEpCb(void* param, SDataBuf* pMsg, int32_t code) { pParam->code = code; if (code != 0) { - tscError("consumer:0x%" PRIx64 ", get topic endpoint error, async:%d, code:%s", tmq->consumerId, - pParam->async, tstrerror(code)); + tscError("consumer:0x%" PRIx64 ", get topic endpoint error, async:%d, code:%s", tmq->consumerId, pParam->async, + tstrerror(code)); goto END; } @@ -1511,26 +1507,26 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); if (tlen < 0) { - tscError("consumer:0x%"PRIx64", tSerializeSMqAskEpReq failed", tmq->consumerId); + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq failed", tmq->consumerId); return -1; } void* pReq = taosMemoryCalloc(1, tlen); if (pReq == NULL) { - tscError("consumer:0x%"PRIx64", failed to malloc askEpReq msg, size:%d", tmq->consumerId, tlen); + tscError("consumer:0x%" PRIx64 ", failed to malloc askEpReq msg, size:%d", tmq->consumerId, tlen); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } if (tSerializeSMqAskEpReq(pReq, tlen, &req) < 0) { - tscError("consumer:0x%"PRIx64", tSerializeSMqAskEpReq %d failed", tmq->consumerId, tlen); + tscError("consumer:0x%" PRIx64 ", tSerializeSMqAskEpReq %d failed", tmq->consumerId, tlen); taosMemoryFree(pReq); return -1; } SMqAskEpCbParam* pParam = taosMemoryCalloc(1, sizeof(SMqAskEpCbParam)); if (pParam == NULL) { - tscError("consumer:0x%"PRIx64", failed to malloc subscribe param", tmq->consumerId); + tscError("consumer:0x%" PRIx64 ", failed to malloc subscribe param", tmq->consumerId); taosMemoryFree(pReq); /*atomic_store_8(&tmq->epStatus, 0);*/ return -1; @@ -1563,7 +1559,8 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%"PRIx64, tmq->consumerId, async, tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " ask ep from mnode, async:%d, reqId:0x%" PRIx64, tmq->consumerId, async, + tmq->consumerId); int64_t transporterId = 0; asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); @@ -1674,7 +1671,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it + pParam->pVg = pVg; // pVg may be released,fix it pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; @@ -1714,20 +1711,19 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p // broadcast the poll request to all related vnodes int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); - tscDebug("consumer:0x%" PRIx64" start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); + tscDebug("consumer:0x%" PRIx64 " start to poll data, numOfTopics:%d", tmq->consumerId, numOfTopics); for (int i = 0; i < numOfTopics; i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - int32_t numOfVg = taosArrayGetSize(pTopic->vgs); + int32_t numOfVg = taosArrayGetSize(pTopic->vgs); for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, - vgSkipCnt); + tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, + pVg->vgId, vgSkipCnt); continue; /*if (vgSkipCnt < 10000) continue;*/ #if 0 @@ -1771,7 +1767,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) } void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { - tscDebug("consumer:0x%"PRIx64" start to handle the rsp", tmq->consumerId); + tscDebug("consumer:0x%" PRIx64 " start to handle the rsp", tmq->consumerId); while (1) { SMqRspWrapper* rspWrapper = NULL; @@ -1811,7 +1807,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("consumer:0x%"PRIx64" msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->dataRsp.head.epoch, consumerEpoch); tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); @@ -1833,7 +1829,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("consumer:0x%"PRIx64" msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); @@ -1864,7 +1860,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("consumer:0x%"PRIx64" msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", + tscDebug("consumer:0x%" PRIx64 " msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", tmq->consumerId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); tmqFreeRspWrapper(rspWrapper); taosFreeQitem(pollRspWrapper); @@ -1910,7 +1906,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { return NULL; } - tscDebug("consumer:0x%"PRIx64" not ready, retry:%d/10 in 500ms", tmq->consumerId, retryCnt); + tscDebug("consumer:0x%" PRIx64 " not ready, retry:%d/10 in 500ms", tmq->consumerId, retryCnt); taosMsleep(500); } } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 54778e87a788190c55fafc0e4d0db732677d795d..76911e229a569403ccdcc36b4b90470d3f02618e 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -25,7 +25,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" -#include "../src/clientSml.c" +#include "../inc/clientSml.h" #include "taos.h" int main(int argc, char **argv) { @@ -40,11 +40,14 @@ TEST(testCase, smlParseInfluxString_Test) { msgBuf.len = 256; SSmlLineInfo elements = {0}; + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; // case 1 char *tmp = "\\,st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 ,32,c=3"; char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, tmp, strlen(tmp) + 1); - int ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + int ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen(",st")); @@ -58,28 +61,23 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 2 false tmp = "st,t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); - - // case 3 false - tmp = "st, t1=3,t2=4,t3=t3 c1=3i64,c3=\"passit hello,c1=2,c2=false,c4=4f64 1626006833639000000"; - memcpy(sql, tmp, strlen(tmp) + 1); - memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); - ASSERT_EQ(elements.cols, sql + elements.measureTagsLen + 1); - ASSERT_EQ(elements.colsLen, strlen("t1=3,t2=4,t3=t3")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 4 tag is null tmp = "st, c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000"; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -93,12 +91,14 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + elements.measureTagsLen + 1 + elements.colsLen + 1); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 5 tag is null tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 1626006833639000000 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); ASSERT_EQ(elements.measure, sql + 1); ASSERT_EQ(elements.measureLen, strlen("st")); @@ -111,91 +111,104 @@ TEST(testCase, smlParseInfluxString_Test) { ASSERT_EQ(elements.timestamp, sql + 1 + elements.measureTagsLen + 3 + elements.colsLen + 2); ASSERT_EQ(elements.timestampLen, strlen("1626006833639000000")); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 6 tmp = " st c1=3i64,c3=\"passit hello,c1=2\",c2=false,c4=4f64 "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_EQ(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + smlClearForRerun(info); // case 7 tmp = " st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); - ASSERT_EQ(ret, 0); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); + ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; // case 8 false tmp = ", st , "; memcpy(sql, tmp, strlen(tmp) + 1); memset(&elements, 0, sizeof(SSmlLineInfo)); - ret = smlParseInfluxString(sql, sql + strlen(sql), &elements, &msgBuf); + ret = smlParseInfluxString(info, sql, sql + strlen(sql), &elements); ASSERT_NE(ret, 0); + taosArrayDestroy(elements.colArray); + elements.colArray = NULL; + taosMemoryFree(sql); + smlDestroyInfo(info); + } TEST(testCase, smlParseCols_Error_Test) { - const char *data[] = {"c=\"89sd", // binary, nchar - "c=j\"89sd\"", - "c=\"89sd\"k", - "c=u", // bool - "c=truet", - "c=f64", // double - "c=8f64f", - "c=8ef64", - "c=f32", // float - "c=8f32f", - "c=8wef32", - "c=-3.402823466e+39f32", - "c=", // double - "c=8f", - "c=8we", - "c=i8", // tiny int - "c=-8i8f", - "c=8wei8", - "c=-999i8", - "c=u8", // u tiny int - "c=8fu8", - "c=8weu8", - "c=999u8", - "c=-8u8", - "c=i16", // small int - "c=8fi16u", - "c=8wei16", - "c=-67787i16", - "c=u16", // u small int - "c=8u16f", - "c=8weu16", - "c=-9u16", - "c=67787u16", - "c=i32", // int - "c=8i32f", - "c=8wei32", - "c=2147483649i32", - "c=u32", // u int - "c=8u32f", - "c=8weu32", - "c=-4u32", - "c=42949672958u32", - "c=i64", // big int - "c=8i64i", - "c=8wei64", - "c=-9223372036854775809i64", - "c=i", // big int - "c=8fi", - "c=8wei", - "c=9223372036854775808i", - "c=u64", // u big int - "c=8u64f", - "c=8weu64", - "c=-3.402823466e+39u64", - "c=-339u64", - "c=18446744073709551616u64", - "c=1,c=2", - "c=1=2"}; - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + const char *data[] = {"st,t=1 c=\"89sd 1626006833639000000", // binary, nchar + "st,t=1 c=j\"89sd\" 1626006833639000000", + "st,t=1 c=\"89sd\"k 1626006833639000000", + "st,t=1 c=u 1626006833639000000", // bool + "st,t=1 c=truet 1626006833639000000", + "st,t=1 c=f64 1626006833639000000", // double + "st,t=1 c=8f64f 1626006833639000000", + "st,t=1 c=8ef64 1626006833639000000", + "st,t=1 c=f32 1626006833639000000", // float + "st,t=1 c=8f32f 1626006833639000000", + "st,t=1 c=8wef32 1626006833639000000", + "st,t=1 c=-3.402823466e+39f32 1626006833639000000", + "st,t=1 c= 1626006833639000000", // double + "st,t=1 c=8f 1626006833639000000", + "st,t=1 c=8we 1626006833639000000", + "st,t=1 c=i8 1626006833639000000", // tiny int + "st,t=1 c=-8i8f 1626006833639000000", + "st,t=1 c=8wei8 1626006833639000000", + "st,t=1 c=-999i8 1626006833639000000", + "st,t=1 c=u8 1626006833639000000", // u tiny int + "st,t=1 c=8fu8 1626006833639000000", + "st,t=1 c=8weu8 1626006833639000000", + "st,t=1 c=999u8 1626006833639000000", + "st,t=1 c=-8u8 1626006833639000000", + "st,t=1 c=i16 1626006833639000000", // small int + "st,t=1 c=8fi16u 1626006833639000000", + "st,t=1 c=8wei16 1626006833639000000", + "st,t=1 c=-67787i16 1626006833639000000", + "st,t=1 c=u16 1626006833639000000", // u small int + "st,t=1 c=8u16f 1626006833639000000", + "st,t=1 c=8weu16 1626006833639000000", + "st,t=1 c=-9u16 1626006833639000000", + "st,t=1 c=67787u16 1626006833639000000", + "st,t=1 c=i32 1626006833639000000", // int + "st,t=1 c=8i32f 1626006833639000000", + "st,t=1 c=8wei32 1626006833639000000", + "st,t=1 c=2147483649i32 1626006833639000000", + "st,t=1 c=u32 1626006833639000000", // u int + "st,t=1 c=8u32f 1626006833639000000", + "st,t=1 c=8weu32 1626006833639000000", + "st,t=1 c=-4u32 1626006833639000000", + "st,t=1 c=42949672958u32 1626006833639000000", + "st,t=1 c=i64 1626006833639000000", // big int + "st,t=1 c=8i64i 1626006833639000000", + "st,t=1 c=8wei64 1626006833639000000", + "st,t=1 c=-9223372036854775809i64 1626006833639000000", + "st,t=1 c=i 1626006833639000000", // big int + "st,t=1 c=8fi 1626006833639000000", + "st,t=1 c=8wei 1626006833639000000", + "st,t=1 c=9223372036854775808i 1626006833639000000", + "st,t=1 c=u64 1626006833639000000", // u big int + "st,t=1 c=8u64f 1626006833639000000", + "st,t=1 c=8weu64 1626006833639000000", + "st,t=1 c=-3.402823466e+39u64 1626006833639000000", + "st,t=1 c=-339u64 1626006833639000000", + "st,t=1 c=18446744073709551616u64 1626006833639000000", + "st,t=1 c=1=2 1626006833639000000"}; + + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; for (int i = 0; i < sizeof(data) / sizeof(data[0]); i++) { char msg[256] = {0}; SSmlMsgBuf msgBuf; @@ -204,76 +217,14 @@ TEST(testCase, smlParseCols_Error_Test) { int32_t len = strlen(data[i]); char *sql = (char *)taosMemoryCalloc(256, 1); memcpy(sql, data[i], len + 1); - SArray *cols = taosArrayInit(8, POINTER_BYTES); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); - printf("i:%d\n", i); + SSmlLineInfo elements = {0}; + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, TSDB_CODE_SUCCESS); - taosHashClear(dumplicateKey); taosMemoryFree(sql); - for (int j = 0; j < taosArrayGetSize(cols); j++) { - void *kv = taosArrayGetP(cols, j); - taosMemoryFree(kv); - } - taosArrayDestroy(cols); - } - taosHashCleanup(dumplicateKey); -} - -TEST(testCase, smlParseCols_tag_Test) { - char msg[256] = {0}; - SSmlMsgBuf msgBuf; - msgBuf.buf = msg; - msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - - const char *data = - "cbin=\"passit " - "helloc\",cnch=L\"iisdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=233i16,cu16=" - "898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t,cboolf=f,cnch_" - "=l\"iuwq\""; - int32_t len = strlen(data); - int32_t ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); - - // nchar - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); - ASSERT_EQ(strncasecmp(kv->key, "cbin", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 15); - ASSERT_EQ(strncasecmp(kv->value, "\"passit", 7), 0); - - // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 3); - ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); - ASSERT_EQ(kv->keyLen, 4); - ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); - ASSERT_EQ(kv->length, 7); - ASSERT_EQ(strncasecmp(kv->value, "4.31f64", 7), 0); - - for (int i = 0; i < size; i++) { - void *tmp = taosArrayGetP(cols, i); - taosMemoryFree(tmp); + taosArrayDestroy(elements.colArray); } - taosArrayClear(cols); - - // test tag is null - data = "t=3e"; - len = 0; - memset(msgBuf.buf, 0, msgBuf.len); - taosHashClear(dumplicateKey); - ret = smlParseCols(data, len, cols, NULL, true, dumplicateKey, &msgBuf); - ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - size = taosArrayGetSize(cols); - ASSERT_EQ(size, 0); - - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + smlDestroyInfo(info); } TEST(testCase, smlParseCols_Test) { @@ -281,202 +232,183 @@ TEST(testCase, smlParseCols_Test) { SSmlMsgBuf msgBuf; msgBuf.buf = msg; msgBuf.len = 256; - - SArray *cols = taosArrayInit(16, POINTER_BYTES); - ASSERT_NE(cols, nullptr); - - SHashObj *dumplicateKey = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->protocol = TSDB_SML_LINE_PROTOCOL; + info->dataFormat = false; + SSmlLineInfo elements = {0}; + info->msgBuf = msgBuf; const char *data = - "cb\\=in=\"pass\\,it " + "st,t=1 cb\\=in=\"pass\\,it " "hello,c=2\",cnch=L\"ii\\=sdfsf\",cbool=false,cf64=4.31f64,cf64_=8.32,cf32=8.23f32,ci8=-34i8,cu8=89u8,ci16=" "233i16,cu16=898u16,ci32=98289i32,cu32=12323u32,ci64=-89238i64,ci=989i,cu64=8989323u64,cbooltrue=true,cboolt=t," - "cboolf=f,cnch_=l\"iuwq\""; + "cboolf=f,cnch_=l\"iuwq\" 1626006833639000000"; int32_t len = strlen(data); char *sql = (char *)taosMemoryCalloc(1024, 1); memcpy(sql, data, len + 1); - int32_t ret = smlParseCols(sql, len, cols, NULL, false, dumplicateKey, &msgBuf); + int32_t ret = smlParseInfluxString(info, sql, sql + len, &elements); ASSERT_EQ(ret, TSDB_CODE_SUCCESS); - int32_t size = taosArrayGetSize(cols); - ASSERT_EQ(size, 19); + int32_t size = taosArrayGetSize(elements.colArray); + ASSERT_EQ(size, 20); // binary - SSmlKv *kv = (SSmlKv *)taosArrayGetP(cols, 0); + SSmlKv *kv = (SSmlKv *)taosArrayGet(elements.colArray, 1); ASSERT_EQ(strncasecmp(kv->key, "cb=in", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BINARY); ASSERT_EQ(kv->length, 17); ASSERT_EQ(strncasecmp(kv->value, "pass,it ", 8), 0); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 1); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 2); ASSERT_EQ(strncasecmp(kv->key, "cnch", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 8); ASSERT_EQ(strncasecmp(kv->value, "ii=sd", 5), 0); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 2); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 3); ASSERT_EQ(strncasecmp(kv->key, "cbool", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // double - kv = (SSmlKv *)taosArrayGetP(cols, 3); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 4); ASSERT_EQ(strncasecmp(kv->key, "cf64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->d, 4.31); printf("4.31 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 4); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 5); ASSERT_EQ(strncasecmp(kv->key, "cf64_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(kv->length, 8); // ASSERT_EQ(kv->f, 8.32); printf("8.32 = kv->d:%f\n", kv->d); - taosMemoryFree(kv); // float - kv = (SSmlKv *)taosArrayGetP(cols, 5); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 6); ASSERT_EQ(strncasecmp(kv->key, "cf32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_FLOAT); ASSERT_EQ(kv->length, 4); // ASSERT_EQ(kv->f, 8.23); printf("8.23 = kv->f:%f\n", kv->f); - taosMemoryFree(kv); // tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 6); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 7); ASSERT_EQ(strncasecmp(kv->key, "ci8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_TINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, -34); - taosMemoryFree(kv); // unsigned tiny int - kv = (SSmlKv *)taosArrayGetP(cols, 7); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 8); ASSERT_EQ(strncasecmp(kv->key, "cu8", 3), 0); ASSERT_EQ(kv->keyLen, 3); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UTINYINT); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->u, 89); - taosMemoryFree(kv); // small int - kv = (SSmlKv *)taosArrayGetP(cols, 8); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 9); ASSERT_EQ(strncasecmp(kv->key, "ci16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_SMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 233); - taosMemoryFree(kv); // unsigned smallint - kv = (SSmlKv *)taosArrayGetP(cols, 9); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 10); ASSERT_EQ(strncasecmp(kv->key, "cu16", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_USMALLINT); ASSERT_EQ(kv->length, 2); ASSERT_EQ(kv->u, 898); - taosMemoryFree(kv); // int - kv = (SSmlKv *)taosArrayGetP(cols, 10); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 11); ASSERT_EQ(strncasecmp(kv->key, "ci32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_INT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 98289); - taosMemoryFree(kv); // unsigned int - kv = (SSmlKv *)taosArrayGetP(cols, 11); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 12); ASSERT_EQ(strncasecmp(kv->key, "cu32", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UINT); ASSERT_EQ(kv->length, 4); ASSERT_EQ(kv->u, 12323); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 12); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 13); ASSERT_EQ(strncasecmp(kv->key, "ci64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, -89238); - taosMemoryFree(kv); // bigint - kv = (SSmlKv *)taosArrayGetP(cols, 13); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 14); ASSERT_EQ(strncasecmp(kv->key, "ci", 2), 0); ASSERT_EQ(kv->keyLen, 2); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->i, 989); - taosMemoryFree(kv); // unsigned bigint - kv = (SSmlKv *)taosArrayGetP(cols, 14); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 15); ASSERT_EQ(strncasecmp(kv->key, "cu64", 4), 0); ASSERT_EQ(kv->keyLen, 4); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_UBIGINT); ASSERT_EQ(kv->length, 8); ASSERT_EQ(kv->u, 8989323); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 15); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 16); ASSERT_EQ(strncasecmp(kv->key, "cbooltrue", 9), 0); ASSERT_EQ(kv->keyLen, 9); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 16); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 17); ASSERT_EQ(strncasecmp(kv->key, "cboolt", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, true); - taosMemoryFree(kv); // bool - kv = (SSmlKv *)taosArrayGetP(cols, 17); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 18); ASSERT_EQ(strncasecmp(kv->key, "cboolf", 6), 0); ASSERT_EQ(kv->keyLen, 6); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(kv->length, 1); ASSERT_EQ(kv->i, false); - taosMemoryFree(kv); // nchar - kv = (SSmlKv *)taosArrayGetP(cols, 18); + kv = (SSmlKv *)taosArrayGet(elements.colArray, 19); ASSERT_EQ(strncasecmp(kv->key, "cnch_", 5), 0); ASSERT_EQ(kv->keyLen, 5); ASSERT_EQ(kv->type, TSDB_DATA_TYPE_NCHAR); ASSERT_EQ(kv->length, 4); ASSERT_EQ(strncasecmp(kv->value, "iuwq", 4), 0); - taosMemoryFree(kv); - taosArrayDestroy(cols); - taosHashCleanup(dumplicateKey); + taosArrayDestroy(elements.colArray); taosMemoryFree(sql); + smlDestroyInfo(info); } TEST(testCase, smlGetTimestampLen_Test) { @@ -515,7 +447,9 @@ TEST(testCase, smlParseNumber_Test) { } TEST(testCase, smlParseTelnetLine_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { @@ -532,479 +466,95 @@ TEST(testCase, smlParseTelnetLine_error_Test) { "sys.procs.running 1479496100 42 ", "sys.procs.running 1479496100 42 host= ", "sys.procs.running 1479496100 42or host=web01", - "sys.procs.running 1479496100 true host=web01", - "sys.procs.running 1479496100 \"binary\" host=web01", - "sys.procs.running 1479496100 L\"rfr\" host=web01", +// "sys.procs.running 1479496100 true host=web01", +// "sys.procs.running 1479496100 \"binary\" host=web01", +// "sys.procs.running 1479496100 L\"rfr\" host=web01", "sys.procs.running 1479496100 42 host=web01 cpu= ", - "sys.procs.running 1479496100 42 host=web01 host=w2", "sys.procs.running 1479496100 42 host=web01 host", "sys.procs.running 1479496100 42 host=web01=er", "sys.procs.running 1479496100 42 host= web01", }; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - int ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); ASSERT_NE(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_type_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = {"sys.procs.running 1479496104000 42 host=web01", - "sys.procs.running 1479496104000 42u8 host=web01", - "appywjnuct 1626006833641 True id=\"appywjnuct_40601_49808_1\" t0=t t1=127i8 " - "id=\"appywjnuct_40601_49808_2\" t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 " - "t5=11.12345f32 t6=22.123456789f64 t7=\"binaryTagValue\" t8=L\"ncharTagValue\""}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_json_error_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); +TEST(testCase, smlParseTelnetLine_Test) { + SSmlHandle *info = smlBuildSmlInfo(NULL); + info->dataFormat = false; + info->protocol = TSDB_SML_TELNET_PROTOCOL; ASSERT_NE(info, nullptr); const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 13468464009999333322222223,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400i,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"web01\",\n" - " \"dc\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"groupid\": { \n" - " \"value\" : 2,\n" - " \"type\" : \"nchar\"\n" - " },\n" - " \"location\": { \n" - " \"value\" : \"北京\",\n" - " \"type\" : \"binary\"\n" - " },\n" - " \"id\": \"d1001\"\n" - " }\n" - " },\n" - "]", + "twudyr 1626006833641 \"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" id=twudyr_17102_17825 t0=t t1=127i8 t2=32767i16 t3=2147483647i32 t4=9223372036854775807i64 t5=11.12345f32 t6=22.123456789f64 t7=\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\" t8=L\"abcd`~!@#$%^&*()_-{[}]|:;<.>?lfjal\"", }; - - int ret = TSDB_CODE_SUCCESS; for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - ASSERT_NE(ret, 0); + SSmlLineInfo elements = {0}; + int ret = smlParseTelnetString(info, (char*)sql[i], (char*)(sql[i] + strlen(sql[i])), &elements); +// printf("i:%d\n", i); + ASSERT_EQ(ret, 0); } smlDestroyInfo(info); } -TEST(testCase, smlParseTelnetLine_diff_json_type1_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": 8\n" - " }\n" - " },\n" - "]", - }; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": 18,\n" - " \"tags\": {\n" - " \"host\": \"lga\"\n" - " }\n" - " },\n" - "]", - "[\n" - " {\n" - " \"metric\": \"sys.cpu.nice\",\n" - " \"timestamp\": 1346846400,\n" - " \"value\": \"18\",\n" - " \"tags\": {\n" - " \"host\": \"fff\"\n" - " }\n" - " },\n" - "]", - }; - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseTelnetLine(info, (void *)sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; - } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} - -TEST(testCase, sml_col_4096_Test) { - SSmlHandle *info = smlBuildSmlInfo(NULL, NULL, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - ASSERT_NE(info, nullptr); - - const char *sql[] = { - "spgwgvldxv,id=spgwgvldxv_1,t0=f " - "c0=t,c1=t,c2=t,c3=t,c4=t,c5=t,c6=t,c7=t,c8=t,c9=t,c10=t,c11=t,c12=t,c13=t,c14=t,c15=t,c16=t,c17=t,c18=t,c19=t," - "c20=t,c21=t,c22=t,c23=t,c24=t,c25=t,c26=t,c27=t,c28=t,c29=t,c30=t,c31=t,c32=t,c33=t,c34=t,c35=t,c36=t,c37=t,c38=" - "t,c39=t,c40=t,c41=t,c42=t,c43=t,c44=t,c45=t,c46=t,c47=t,c48=t,c49=t,c50=t,c51=t,c52=t,c53=t,c54=t,c55=t,c56=t," - "c57=t,c58=t,c59=t,c60=t,c61=t,c62=t,c63=t,c64=t,c65=t,c66=t,c67=t,c68=t,c69=t,c70=t,c71=t,c72=t,c73=t,c74=t,c75=" - "t,c76=t,c77=t,c78=t,c79=t,c80=t,c81=t,c82=t,c83=t,c84=t,c85=t,c86=t,c87=t,c88=t,c89=t,c90=t,c91=t,c92=t,c93=t," - "c94=t,c95=t,c96=t,c97=t,c98=t,c99=t,c100=t," - "c101=t,c102=t,c103=t,c104=t,c105=t,c106=t,c107=t,c108=t,c109=t,c110=t,c111=t,c112=t,c113=t,c114=t,c115=t,c116=t," - "c117=t,c118=t,c119=t,c120=t,c121=t,c122=t,c123=t,c124=t,c125=t,c126=t,c127=t,c128=t,c129=t,c130=t,c131=t,c132=t," - "c133=t,c134=t,c135=t,c136=t,c137=t,c138=t,c139=t,c140=t,c141=t,c142=t,c143=t,c144=t,c145=t,c146=t,c147=t,c148=t," - "c149=t,c150=t,c151=t,c152=t,c153=t,c154=t,c155=t,c156=t,c157=t,c158=t,c159=t,c160=t,c161=t,c162=t,c163=t,c164=t," - "c165=t,c166=t,c167=t,c168=t,c169=t,c170=t,c171=t,c172=t,c173=t,c174=t,c175=t,c176=t,c177=t,c178=t,c179=t,c180=t," - "c181=t,c182=t,c183=t,c184=t,c185=t,c186=t,c187=t,c188=t,c189=t," - "c190=t,c191=t,c192=t,c193=t,c194=t,c195=t,c196=t,c197=t,c198=t,c199=t,c200=t,c201=t,c202=t,c203=t,c204=t,c205=t," - "c206=t,c207=t,c208=t,c209=t,c210=t,c211=t,c212=t,c213=t,c214=t,c215=t,c216=t,c217=t,c218=t,c219=t,c220=t,c221=t," - "c222=t,c223=t,c224=t,c225=t,c226=t,c227=t,c228=t,c229=t,c230=t,c231=t,c232=t,c233=t,c234=t,c235=t,c236=t,c237=t," - "c238=t,c239=t,c240=t,c241=t,c242=t,c243=t,c244=t,c245=t,c246=t,c247=t,c248=t,c249=t,c250=t,c251=t,c252=t,c253=t," - "c254=t,c255=t,c256=t,c257=t,c258=t,c259=t,c260=t,c261=t,c262=t,c263=t,c264=t,c265=t,c266=t,c267=t,c268=t,c269=t," - "c270=t,c271=t,c272=t,c273=t,c274=t,c275=t,c276=t,c277=t,c278=t," - "c279=t,c280=t,c281=t,c282=t,c283=t,c284=t,c285=t,c286=t,c287=t,c288=t,c289=t,c290=t,c291=t,c292=t,c293=t,c294=t," - "c295=t,c296=t,c297=t,c298=t,c299=t,c300=t,c301=t,c302=t,c303=t,c304=t,c305=t,c306=t,c307=t,c308=t,c309=t,c310=t," - "c311=t,c312=t,c313=t,c314=t,c315=t,c316=t,c317=t,c318=t,c319=t,c320=t,c321=t,c322=t,c323=t,c324=t,c325=t,c326=t," - "c327=t,c328=t,c329=t,c330=t,c331=t,c332=t,c333=t,c334=t,c335=t,c336=t,c337=t,c338=t,c339=t,c340=t,c341=t,c342=t," - "c343=t,c344=t,c345=t,c346=t,c347=t,c348=t,c349=t,c350=t,c351=t,c352=t,c353=t,c354=t,c355=t,c356=t,c357=t,c358=t," - "c359=t,c360=t,c361=t,c362=t,c363=t,c364=t,c365=t,c366=t,c367=t,c368=t,c369=t,c370=t,c371=t,c372=t,c373=t,c374=t," - "c375=t,c376=t,c377=t,c378=t,c379=t,c380=t,c381=t,c382=t,c383=t,c384=t,c385=t,c386=t,c387=t,c388=t,c389=t,c390=t," - "c391=t,c392=t,c393=t,c394=t,c395=t,c396=t,c397=t,c398=t,c399=t,c400=t,c401=t,c402=t,c403=t,c404=t,c405=t,c406=t," - "c407=t,c408=t,c409=t,c410=t,c411=t,c412=t,c413=t,c414=t,c415=t,c416=t,c417=t,c418=t,c419=t,c420=t,c421=t,c422=t," - "c423=t,c424=t,c425=t,c426=t,c427=t,c428=t,c429=t,c430=t,c431=t,c432=t,c433=t,c434=t,c435=t,c436=t,c437=t,c438=t," - "c439=t,c440=t,c441=t,c442=t,c443=t,c444=t,c445=t,c446=t," - "c447=t,c448=t,c449=t,c450=t,c451=t,c452=t,c453=t,c454=t,c455=t,c456=t,c457=t,c458=t,c459=t,c460=t,c461=t,c462=t," - "c463=t,c464=t,c465=t,c466=t,c467=t,c468=t,c469=t,c470=t,c471=t,c472=t,c473=t,c474=t,c475=t,c476=t,c477=t,c478=t," - "c479=t,c480=t,c481=t,c482=t,c483=t,c484=t,c485=t,c486=t,c487=t,c488=t,c489=t,c490=t,c491=t,c492=t,c493=t,c494=t," - "c495=t,c496=t,c497=t,c498=t,c499=t,c500=t,c501=t,c502=t,c503=t,c504=t,c505=t,c506=t,c507=t,c508=t,c509=t,c510=t," - "c511=t,c512=t,c513=t,c514=t,c515=t,c516=t,c517=t,c518=t,c519=t,c520=t,c521=t,c522=t,c523=t,c524=t,c525=t,c526=t," - "c527=t,c528=t,c529=t,c530=t,c531=t,c532=t,c533=t,c534=t,c535=t,c536=t,c537=t,c538=t,c539=t,c540=t,c541=t,c542=t," - "c543=t,c544=t,c545=t,c546=t,c547=t,c548=t,c549=t,c550=t,c551=t,c552=t,c553=t,c554=t,c555=t,c556=t,c557=t,c558=t," - "c559=t,c560=t,c561=t,c562=t,c563=t,c564=t,c565=t,c566=t,c567=t,c568=t,c569=t,c570=t,c571=t,c572=t,c573=t,c574=t," - "c575=t,c576=t,c577=t,c578=t,c579=t,c580=t,c581=t,c582=t,c583=t,c584=t,c585=t,c586=t,c587=t,c588=t,c589=t,c590=t," - "c591=t,c592=t,c593=t,c594=t,c595=t,c596=t,c597=t,c598=t,c599=t,c600=t,c601=t,c602=t,c603=t,c604=t,c605=t,c606=t," - "c607=t,c608=t,c609=t,c610=t,c611=t,c612=t,c613=t,c614=t," - "c615=t,c616=t,c617=t,c618=t,c619=t,c620=t,c621=t,c622=t,c623=t,c624=t,c625=t,c626=t,c627=t,c628=t,c629=t,c630=t," - "c631=t,c632=t,c633=t,c634=t,c635=t,c636=t,c637=t,c638=t,c639=t,c640=t,c641=t,c642=t,c643=t,c644=t,c645=t,c646=t," - "c647=t,c648=t,c649=t,c650=t,c651=t,c652=t,c653=t,c654=t,c655=t,c656=t,c657=t,c658=t,c659=t,c660=t,c661=t,c662=t," - "c663=t,c664=t,c665=t,c666=t,c667=t,c668=t,c669=t,c670=t,c671=t,c672=t,c673=t,c674=t,c675=t,c676=t,c677=t,c678=t," - "c679=t,c680=t,c681=t,c682=t,c683=t,c684=t,c685=t,c686=t,c687=t,c688=t,c689=t,c690=t,c691=t,c692=t,c693=t,c694=t," - "c695=t,c696=t,c697=t,c698=t,c699=t,c700=t,c701=t,c702=t,c703=t,c704=t,c705=t,c706=t,c707=t,c708=t,c709=t,c710=t," - "c711=t,c712=t,c713=t,c714=t,c715=t,c716=t,c717=t,c718=t,c719=t,c720=t,c721=t,c722=t,c723=t,c724=t,c725=t,c726=t," - "c727=t,c728=t,c729=t,c730=t,c731=t,c732=t,c733=t,c734=t,c735=t,c736=t,c737=t,c738=t,c739=t,c740=t,c741=t,c742=t," - "c743=t,c744=t,c745=t,c746=t,c747=t,c748=t,c749=t,c750=t,c751=t,c752=t,c753=t,c754=t,c755=t,c756=t,c757=t,c758=t," - "c759=t,c760=t,c761=t,c762=t,c763=t,c764=t,c765=t,c766=t,c767=t,c768=t,c769=t,c770=t,c771=t,c772=t,c773=t,c774=t," - "c775=t,c776=t,c777=t,c778=t,c779=t,c780=t,c781=t,c782=t," - "c783=t,c784=t,c785=t,c786=t,c787=t,c788=t,c789=t,c790=t,c791=t,c792=t,c793=t,c794=t,c795=t,c796=t,c797=t,c798=t," - "c799=t,c800=t,c801=t,c802=t,c803=t,c804=t,c805=t,c806=t,c807=t,c808=t,c809=t,c810=t,c811=t,c812=t,c813=t," - "c814=t,c815=t,c816=t,c817=t,c818=t,c819=t,c820=t,c821=t,c822=t,c823=t,c824=t,c825=t,c826=t,c827=t,c828=t,c829=t," - "c830=t,c831=t,c832=t,c833=t,c834=t,c835=t,c836=t,c837=t,c838=t,c839=t,c840=t,c841=t,c842=t,c843=t,c844=t,c845=t," - "c846=t,c847=t,c848=t,c849=t,c850=t,c851=t,c852=t,c853=t,c854=t,c855=t,c856=t,c857=t,c858=t,c859=t,c860=t,c861=t," - "c862=t," - "c863=t,c864=t,c865=t,c866=t,c867=t,c868=t,c869=t,c870=t,c871=t,c872=t,c873=t,c874=t,c875=t,c876=t,c877=t,c878=t," - "c879=t,c880=t,c881=t,c882=t,c883=t,c884=t,c885=t,c886=t,c887=t,c888=t,c889=t,c890=t,c891=t,c892=t,c893=t,c894=t," - "c895=t,c896=t,c897=t,c898=t,c899=t,c900=t,c901=t,c902=t,c903=t,c904=t,c905=t,c906=t,c907=t,c908=t,c909=t,c910=t," - "c911=t,c912=t,c913=t,c914=t,c915=t,c916=t,c917=t,c918=t,c919=t,c920=t,c921=t,c922=t,c923=t,c924=t,c925=t,c926=t," - "c927=t,c928=t,c929=t,c930=t,c931=t,c932=t,c933=t,c934=t,c935=t,c936=t,c937=t,c938=t,c939=t,c940=t,c941=t,c942=t," - "c943=t,c944=t,c945=t,c946=t,c947=t,c948=t,c949=t,c950=t,c951=t,c952=t,c953=t,c954=t,c955=t,c956=t,c957=t,c958=t," - "c959=t,c960=t,c961=t,c962=t,c963=t,c964=t,c965=t,c966=t,c967=t,c968=t,c969=t,c970=t,c971=t,c972=t,c973=t,c974=t," - "c975=t,c976=t,c977=t,c978=t,c979=t,c980=t,c981=t,c982=t,c983=t,c984=t,c985=t,c986=t,c987=t,c988=t,c989=t,c990=t," - "c991=t,c992=t,c993=t,c994=t,c995=t,c996=t,c997=t,c998=t,c999=t,c1000=t,c1001=t,c1002=t,c1003=t,c1004=t,c1005=t," - "c1006=t,c1007=t,c1008=t,c1009=t,c1010=t,c1011=t,c1012=t,c1013=t,c1014=t,c1015=t,c1016=t,c1017=t,c1018=t,c1019=t," - "c1020=t,c1021=t,c1022=t,c1023=t,c1024=t,c1025=t,c1026=t," - "c1027=t,c1028=t,c1029=t,c1030=t,c1031=t,c1032=t,c1033=t,c1034=t,c1035=t,c1036=t,c1037=t,c1038=t,c1039=t,c1040=t," - "c1041=t,c1042=t,c1043=t,c1044=t,c1045=t,c1046=t,c1047=t,c1048=t,c1049=t,c1050=t,c1051=t,c1052=t,c1053=t,c1054=t," - "c1055=t,c1056=t,c1057=t,c1058=t,c1059=t,c1060=t,c1061=t,c1062=t,c1063=t,c1064=t,c1065=t,c1066=t,c1067=t,c1068=t," - "c1069=t,c1070=t,c1071=t,c1072=t,c1073=t,c1074=t,c1075=t,c1076=t,c1077=t,c1078=t,c1079=t,c1080=t,c1081=t,c1082=t," - "c1083=t,c1084=t,c1085=t,c1086=t,c1087=t,c1088=t,c1089=t,c1090=t,c1091=t,c1092=t,c1093=t,c1094=t,c1095=t,c1096=t," - "c1097=t,c1098=t,c1099=t,c1100=t,c1101=t,c1102=t,c1103=t,c1104=t,c1105=t,c1106=t,c1107=t,c1108=t,c1109=t,c1110=t," - "c1111=t,c1112=t,c1113=t,c1114=t,c1115=t,c1116=t,c1117=t,c1118=t,c1119=t,c1120=t,c1121=t,c1122=t,c1123=t,c1124=t," - "c1125=t,c1126=t,c1127=t,c1128=t,c1129=t,c1130=t,c1131=t,c1132=t,c1133=t,c1134=t,c1135=t,c1136=t,c1137=t,c1138=t," - "c1139=t,c1140=t,c1141=t,c1142=t,c1143=t,c1144=t,c1145=t,c1146=t,c1147=t,c1148=t,c1149=t,c1150=t,c1151=t,c1152=t," - "c1153=t,c1154=t,c1155=t,c1156=t,c1157=t,c1158=t,c1159=t,c1160=t,c1161=t,c1162=t,c1163=t,c1164=t,c1165=t,c1166=t," - "c1167=t,c1168=t,c1169=t,c1170=t,c1171=t,c1172=t,c1173=t," - "c1174=t,c1175=t,c1176=t,c1177=t,c1178=t,c1179=t,c1180=t,c1181=t,c1182=t,c1183=t,c1184=t,c1185=t,c1186=t,c1187=t," - "c1188=t,c1189=t,c1190=t,c1191=t,c1192=t,c1193=t,c1194=t,c1195=t,c1196=t,c1197=t,c1198=t,c1199=t,c1200=t,c1201=t," - "c1202=t,c1203=t,c1204=t,c1205=t,c1206=t,c1207=t,c1208=t,c1209=t,c1210=t,c1211=t,c1212=t,c1213=t,c1214=t,c1215=t," - "c1216=t,c1217=t,c1218=t,c1219=t,c1220=t,c1221=t,c1222=t,c1223=t,c1224=t,c1225=t,c1226=t,c1227=t,c1228=t,c1229=t," - "c1230=t,c1231=t,c1232=t,c1233=t,c1234=t,c1235=t,c1236=t,c1237=t,c1238=t,c1239=t,c1240=t,c1241=t,c1242=t,c1243=t," - "c1244=t,c1245=t,c1246=t,c1247=t,c1248=t,c1249=t,c1250=t,c1251=t,c1252=t,c1253=t,c1254=t,c1255=t,c1256=t,c1257=t," - "c1258=t,c1259=t,c1260=t,c1261=t,c1262=t,c1263=t,c1264=t,c1265=t,c1266=t,c1267=t,c1268=t,c1269=t,c1270=t,c1271=t," - "c1272=t,c1273=t,c1274=t,c1275=t,c1276=t,c1277=t,c1278=t,c1279=t,c1280=t,c1281=t,c1282=t,c1283=t,c1284=t,c1285=t," - "c1286=t,c1287=t,c1288=t,c1289=t,c1290=t,c1291=t,c1292=t,c1293=t,c1294=t,c1295=t,c1296=t,c1297=t,c1298=t,c1299=t," - "c1300=t,c1301=t,c1302=t,c1303=t,c1304=t,c1305=t,c1306=t,c1307=t,c1308=t,c1309=t,c1310=t,c1311=t,c1312=t,c1313=t," - "c1314=t,c1315=t,c1316=t,c1317=t,c1318=t,c1319=t,c1320=t," - "c1321=t,c1322=t,c1323=t,c1324=t,c1325=t,c1326=t,c1327=t,c1328=t,c1329=t,c1330=t,c1331=t,c1332=t,c1333=t,c1334=t," - "c1335=t,c1336=t,c1337=t,c1338=t,c1339=t,c1340=t,c1341=t,c1342=t,c1343=t,c1344=t,c1345=t,c1346=t,c1347=t," - "c1348=t,c1349=t,c1350=t,c1351=t,c1352=t,c1353=t,c1354=t,c1355=t,c1356=t,c1357=t,c1358=t,c1359=t,c1360=t,c1361=t," - "c1362=t,c1363=t,c1364=t,c1365=t,c1366=t,c1367=t,c1368=t,c1369=t,c1370=t,c1371=t,c1372=t,c1373=t,c1374=t,c1375=t," - "c1376=t,c1377=t,c1378=t,c1379=t,c1380=t,c1381=t,c1382=t,c1383=t,c1384=t,c1385=t,c1386=t,c1387=t,c1388=t,c1389=t," - "c1390=t,c1391=t,c1392=t,c1393=t,c1394=t,c1395=t,c1396=t,c1397=t,c1398=t,c1399=t,c1400=t,c1401=t,c1402=t,c1403=t," - "c1404=t,c1405=t,c1406=t,c1407=t,c1408=t,c1409=t,c1410=t,c1411=t,c1412=t,c1413=t,c1414=t,c1415=t,c1416=t,c1417=t," - "c1418=t,c1419=t,c1420=t,c1421=t,c1422=t,c1423=t,c1424=t,c1425=t,c1426=t,c1427=t,c1428=t,c1429=t,c1430=t,c1431=t," - "c1432=t,c1433=t,c1434=t,c1435=t,c1436=t,c1437=t,c1438=t,c1439=t,c1440=t,c1441=t,c1442=t,c1443=t,c1444=t,c1445=t," - "c1446=t,c1447=t,c1448=t,c1449=t,c1450=t,c1451=t,c1452=t,c1453=t,c1454=t,c1455=t,c1456=t,c1457=t,c1458=t,c1459=t," - "c1460=t,c1461=t,c1462=t,c1463=t,c1464=t,c1465=t,c1466=t,c1467=t,c1468=t,c1469=t,c1470=t,c1471=t,c1472=t,c1473=t," - "c1474=t,c1475=t,c1476=t,c1477=t,c1478=t,c1479=t,c1480=t,c1481=t,c1482=t,c1483=t,c1484=t,c1485=t,c1486=t,c1487=t," - "c1488=t,c1489=t,c1490=t,c1491=t,c1492=t,c1493=t,c1494=t," - "c1495=t,c1496=t,c1497=t,c1498=t,c1499=t,c1500=t,c1501=t,c1502=t,c1503=t,c1504=t,c1505=t,c1506=t,c1507=t,c1508=t," - "c1509=t,c1510=t,c1511=t,c1512=t,c1513=t,c1514=t,c1515=t,c1516=t,c1517=t,c1518=t,c1519=t,c1520=t,c1521=t,c1522=t," - "c1523=t,c1524=t,c1525=t,c1526=t,c1527=t,c1528=t,c1529=t,c1530=t,c1531=t,c1532=t,c1533=t,c1534=t,c1535=t,c1536=t," - "c1537=t,c1538=t,c1539=t,c1540=t,c1541=t,c1542=t,c1543=t,c1544=t,c1545=t,c1546=t,c1547=t,c1548=t,c1549=t,c1550=t," - "c1551=t,c1552=t,c1553=t,c1554=t,c1555=t,c1556=t,c1557=t,c1558=t,c1559=t,c1560=t,c1561=t,c1562=t,c1563=t,c1564=t," - "c1565=t,c1566=t,c1567=t,c1568=t,c1569=t,c1570=t,c1571=t,c1572=t,c1573=t,c1574=t,c1575=t,c1576=t,c1577=t,c1578=t," - "c1579=t,c1580=t,c1581=t,c1582=t,c1583=t,c1584=t,c1585=t,c1586=t,c1587=t,c1588=t,c1589=t,c1590=t,c1591=t,c1592=t," - "c1593=t,c1594=t,c1595=t,c1596=t,c1597=t,c1598=t,c1599=t,c1600=t,c1601=t,c1602=t,c1603=t,c1604=t,c1605=t,c1606=t," - "c1607=t,c1608=t,c1609=t,c1610=t,c1611=t,c1612=t,c1613=t,c1614=t,c1615=t,c1616=t,c1617=t,c1618=t,c1619=t,c1620=t," - "c1621=t,c1622=t,c1623=t,c1624=t,c1625=t,c1626=t,c1627=t,c1628=t,c1629=t,c1630=t,c1631=t,c1632=t,c1633=t,c1634=t," - "c1635=t,c1636=t,c1637=t,c1638=t,c1639=t,c1640=t,c1641=t," - "c1642=t,c1643=t,c1644=t,c1645=t,c1646=t,c1647=t,c1648=t,c1649=t,c1650=t,c1651=t,c1652=t,c1653=t,c1654=t,c1655=t," - "c1656=t,c1657=t,c1658=t,c1659=t,c1660=t,c1661=t,c1662=t,c1663=t,c1664=t,c1665=t,c1666=t,c1667=t,c1668=t,c1669=t," - "c1670=t,c1671=t,c1672=t,c1673=t,c1674=t,c1675=t,c1676=t,c1677=t,c1678=t,c1679=t,c1680=t,c1681=t,c1682=t,c1683=t," - "c1684=t,c1685=t,c1686=t,c1687=t,c1688=t,c1689=t,c1690=t,c1691=t,c1692=t,c1693=t,c1694=t,c1695=t,c1696=t,c1697=t," - "c1698=t,c1699=t,c1700=t,c1701=t,c1702=t,c1703=t,c1704=t,c1705=t,c1706=t,c1707=t,c1708=t,c1709=t,c1710=t,c1711=t," - "c1712=t,c1713=t,c1714=t,c1715=t,c1716=t,c1717=t,c1718=t,c1719=t,c1720=t,c1721=t,c1722=t,c1723=t,c1724=t,c1725=t," - "c1726=t,c1727=t,c1728=t,c1729=t,c1730=t,c1731=t,c1732=t,c1733=t,c1734=t,c1735=t,c1736=t,c1737=t,c1738=t,c1739=t," - "c1740=t,c1741=t,c1742=t,c1743=t,c1744=t,c1745=t,c1746=t,c1747=t,c1748=t,c1749=t,c1750=t,c1751=t,c1752=t,c1753=t," - "c1754=t,c1755=t,c1756=t,c1757=t,c1758=t,c1759=t,c1760=t,c1761=t,c1762=t,c1763=t,c1764=t,c1765=t,c1766=t,c1767=t," - "c1768=t,c1769=t,c1770=t,c1771=t,c1772=t,c1773=t,c1774=t,c1775=t,c1776=t,c1777=t,c1778=t,c1779=t,c1780=t,c1781=t," - "c1782=t,c1783=t,c1784=t,c1785=t,c1786=t,c1787=t,c1788=t," - "c1789=t,c1790=t,c1791=t,c1792=t,c1793=t,c1794=t,c1795=t,c1796=t,c1797=t,c1798=t,c1799=t,c1800=t,c1801=t,c1802=t," - "c1803=t,c1804=t,c1805=t,c1806=t,c1807=t,c1808=t,c1809=t,c1810=t,c1811=t,c1812=t,c1813=t,c1814=t,c1815=t," - "c1816=t,c1817=t,c1818=t,c1819=t,c1820=t,c1821=t,c1822=t,c1823=t,c1824=t,c1825=t,c1826=t,c1827=t,c1828=t,c1829=t," - "c1830=t,c1831=t,c1832=t,c1833=t,c1834=t,c1835=t,c1836=t,c1837=t,c1838=t,c1839=t,c1840=t,c1841=t,c1842=t,c1843=t," - "c1844=t,c1845=t,c1846=t,c1847=t,c1848=t,c1849=t,c1850=t,c1851=t,c1852=t,c1853=t,c1854=t,c1855=t,c1856=t,c1857=t," - "c1858=t,c1859=t,c1860=t,c1861=t,c1862=t,c1863=t,c1864=t,c1865=t,c1866=t,c1867=t,c1868=t,c1869=t,c1870=t,c1871=t," - "c1872=t,c1873=t,c1874=t,c1875=t,c1876=t,c1877=t,c1878=t,c1879=t,c1880=t,c1881=t,c1882=t,c1883=t,c1884=t,c1885=t," - "c1886=t,c1887=t,c1888=t,c1889=t,c1890=t,c1891=t,c1892=t,c1893=t,c1894=t,c1895=t,c1896=t,c1897=t,c1898=t,c1899=t," - "c1900=t,c1901=t,c1902=t,c1903=t,c1904=t,c1905=t,c1906=t,c1907=t,c1908=t,c1909=t,c1910=t,c1911=t,c1912=t,c1913=t," - "c1914=t,c1915=t,c1916=t,c1917=t,c1918=t,c1919=t,c1920=t,c1921=t,c1922=t,c1923=t,c1924=t,c1925=t,c1926=t,c1927=t," - "c1928=t,c1929=t,c1930=t,c1931=t,c1932=t,c1933=t,c1934=t,c1935=t,c1936=t,c1937=t,c1938=t,c1939=t,c1940=t,c1941=t," - "c1942=t,c1943=t,c1944=t,c1945=t,c1946=t,c1947=t,c1948=t,c1949=t,c1950=t,c1951=t,c1952=t,c1953=t,c1954=t,c1955=t," - "c1956=t,c1957=t,c1958=t,c1959=t,c1960=t,c1961=t,c1962=t," - "c1963=t,c1964=t,c1965=t,c1966=t,c1967=t,c1968=t,c1969=t,c1970=t,c1971=t,c1972=t,c1973=t,c1974=t,c1975=t,c1976=t," - "c1977=t,c1978=t,c1979=t,c1980=t,c1981=t,c1982=t,c1983=t,c1984=t,c1985=t,c1986=t,c1987=t,c1988=t,c1989=t,c1990=t," - "c1991=t,c1992=t,c1993=t,c1994=t,c1995=t,c1996=t,c1997=t,c1998=t,c1999=t,c2000=t,c2001=t,c2002=t,c2003=t,c2004=t," - "c2005=t,c2006=t,c2007=t,c2008=t,c2009=t,c2010=t,c2011=t,c2012=t,c2013=t,c2014=t,c2015=t,c2016=t,c2017=t,c2018=t," - "c2019=t,c2020=t,c2021=t,c2022=t,c2023=t,c2024=t,c2025=t,c2026=t,c2027=t,c2028=t,c2029=t,c2030=t,c2031=t,c2032=t," - "c2033=t,c2034=t,c2035=t,c2036=t,c2037=t,c2038=t,c2039=t,c2040=t,c2041=t,c2042=t,c2043=t,c2044=t,c2045=t,c2046=t," - "c2047=t,c2048=t,c2049=t,c2050=t,c2051=t,c2052=t,c2053=t,c2054=t,c2055=t,c2056=t,c2057=t,c2058=t,c2059=t,c2060=t," - "c2061=t,c2062=t,c2063=t,c2064=t,c2065=t,c2066=t,c2067=t,c2068=t,c2069=t,c2070=t,c2071=t,c2072=t,c2073=t,c2074=t," - "c2075=t,c2076=t,c2077=t,c2078=t,c2079=t,c2080=t,c2081=t,c2082=t,c2083=t,c2084=t,c2085=t,c2086=t,c2087=t,c2088=t," - "c2089=t,c2090=t,c2091=t,c2092=t,c2093=t,c2094=t,c2095=t,c2096=t,c2097=t,c2098=t,c2099=t,c2100=t,c2101=t,c2102=t," - "c2103=t,c2104=t,c2105=t,c2106=t,c2107=t,c2108=t,c2109=t," - "c2110=t,c2111=t,c2112=t,c2113=t,c2114=t,c2115=t,c2116=t,c2117=t,c2118=t,c2119=t,c2120=t,c2121=t,c2122=t,c2123=t," - "c2124=t,c2125=t,c2126=t,c2127=t,c2128=t,c2129=t,c2130=t,c2131=t,c2132=t,c2133=t,c2134=t,c2135=t,c2136=t,c2137=t," - "c2138=t,c2139=t,c2140=t,c2141=t,c2142=t,c2143=t,c2144=t,c2145=t,c2146=t,c2147=t,c2148=t,c2149=t,c2150=t,c2151=t," - "c2152=t,c2153=t,c2154=t,c2155=t,c2156=t,c2157=t,c2158=t,c2159=t,c2160=t,c2161=t,c2162=t,c2163=t,c2164=t,c2165=t," - "c2166=t,c2167=t,c2168=t,c2169=t,c2170=t,c2171=t,c2172=t,c2173=t,c2174=t,c2175=t,c2176=t,c2177=t,c2178=t,c2179=t," - "c2180=t,c2181=t,c2182=t,c2183=t,c2184=t,c2185=t,c2186=t,c2187=t,c2188=t,c2189=t,c2190=t,c2191=t,c2192=t,c2193=t," - "c2194=t,c2195=t,c2196=t,c2197=t,c2198=t,c2199=t,c2200=t,c2201=t,c2202=t,c2203=t,c2204=t,c2205=t,c2206=t,c2207=t," - "c2208=t,c2209=t,c2210=t,c2211=t,c2212=t,c2213=t,c2214=t,c2215=t,c2216=t,c2217=t,c2218=t,c2219=t,c2220=t,c2221=t," - "c2222=t,c2223=t,c2224=t,c2225=t,c2226=t,c2227=t,c2228=t,c2229=t,c2230=t,c2231=t,c2232=t,c2233=t,c2234=t,c2235=t," - "c2236=t,c2237=t,c2238=t,c2239=t,c2240=t,c2241=t,c2242=t,c2243=t,c2244=t,c2245=t,c2246=t,c2247=t,c2248=t,c2249=t," - "c2250=t,c2251=t,c2252=t,c2253=t,c2254=t,c2255=t,c2256=t," - "c2257=t,c2258=t,c2259=t,c2260=t,c2261=t,c2262=t,c2263=t,c2264=t,c2265=t,c2266=t,c2267=t,c2268=t,c2269=t,c2270=t," - "c2271=t,c2272=t,c2273=t,c2274=t,c2275=t,c2276=t,c2277=t,c2278=t,c2279=t,c2280=t,c2281=t,c2282=t,c2283=t," - "c2284=t,c2285=t,c2286=t,c2287=t,c2288=t,c2289=t,c2290=t,c2291=t,c2292=t,c2293=t,c2294=t,c2295=t,c2296=t,c2297=t," - "c2298=t,c2299=t,c2300=t,c2301=t,c2302=t,c2303=t,c2304=t,c2305=t,c2306=t,c2307=t,c2308=t,c2309=t,c2310=t,c2311=t," - "c2312=t,c2313=t,c2314=t,c2315=t,c2316=t,c2317=t,c2318=t,c2319=t,c2320=t,c2321=t,c2322=t,c2323=t,c2324=t,c2325=t," - "c2326=t,c2327=t,c2328=t,c2329=t,c2330=t,c2331=t,c2332=t,c2333=t,c2334=t,c2335=t,c2336=t,c2337=t,c2338=t,c2339=t," - "c2340=t,c2341=t,c2342=t,c2343=t,c2344=t,c2345=t,c2346=t,c2347=t,c2348=t,c2349=t,c2350=t,c2351=t,c2352=t,c2353=t," - "c2354=t,c2355=t,c2356=t,c2357=t,c2358=t,c2359=t,c2360=t,c2361=t,c2362=t,c2363=t,c2364=t,c2365=t,c2366=t,c2367=t," - "c2368=t,c2369=t,c2370=t,c2371=t,c2372=t,c2373=t,c2374=t,c2375=t,c2376=t,c2377=t,c2378=t,c2379=t,c2380=t,c2381=t," - "c2382=t,c2383=t,c2384=t,c2385=t,c2386=t,c2387=t,c2388=t,c2389=t,c2390=t,c2391=t,c2392=t,c2393=t,c2394=t,c2395=t," - "c2396=t,c2397=t,c2398=t,c2399=t,c2400=t,c2401=t,c2402=t,c2403=t,c2404=t,c2405=t,c2406=t,c2407=t,c2408=t,c2409=t," - "c2410=t,c2411=t,c2412=t,c2413=t,c2414=t,c2415=t,c2416=t,c2417=t,c2418=t,c2419=t,c2420=t,c2421=t,c2422=t,c2423=t," - "c2424=t,c2425=t,c2426=t,c2427=t,c2428=t,c2429=t,c2430=t," - "c2431=t,c2432=t,c2433=t,c2434=t,c2435=t,c2436=t,c2437=t,c2438=t,c2439=t,c2440=t,c2441=t,c2442=t,c2443=t,c2444=t," - "c2445=t,c2446=t,c2447=t,c2448=t,c2449=t,c2450=t,c2451=t,c2452=t,c2453=t,c2454=t,c2455=t,c2456=t,c2457=t,c2458=t," - "c2459=t,c2460=t,c2461=t,c2462=t,c2463=t,c2464=t,c2465=t,c2466=t,c2467=t,c2468=t,c2469=t,c2470=t,c2471=t,c2472=t," - "c2473=t,c2474=t,c2475=t,c2476=t,c2477=t,c2478=t,c2479=t,c2480=t,c2481=t,c2482=t,c2483=t,c2484=t,c2485=t,c2486=t," - "c2487=t,c2488=t,c2489=t,c2490=t,c2491=t,c2492=t,c2493=t,c2494=t,c2495=t,c2496=t,c2497=t,c2498=t,c2499=t,c2500=t," - "c2501=t,c2502=t,c2503=t,c2504=t,c2505=t,c2506=t,c2507=t,c2508=t,c2509=t,c2510=t,c2511=t,c2512=t,c2513=t,c2514=t," - "c2515=t,c2516=t,c2517=t,c2518=t,c2519=t,c2520=t,c2521=t,c2522=t,c2523=t,c2524=t,c2525=t,c2526=t,c2527=t,c2528=t," - "c2529=t,c2530=t,c2531=t,c2532=t,c2533=t,c2534=t,c2535=t,c2536=t,c2537=t,c2538=t,c2539=t,c2540=t,c2541=t,c2542=t," - "c2543=t,c2544=t,c2545=t,c2546=t,c2547=t,c2548=t,c2549=t,c2550=t,c2551=t,c2552=t,c2553=t,c2554=t,c2555=t,c2556=t," - "c2557=t,c2558=t,c2559=t,c2560=t,c2561=t,c2562=t,c2563=t,c2564=t,c2565=t,c2566=t,c2567=t,c2568=t,c2569=t,c2570=t," - "c2571=t,c2572=t,c2573=t,c2574=t,c2575=t,c2576=t,c2577=t," - "c2578=t,c2579=t,c2580=t,c2581=t,c2582=t,c2583=t,c2584=t,c2585=t,c2586=t,c2587=t,c2588=t,c2589=t,c2590=t,c2591=t," - "c2592=t,c2593=t,c2594=t,c2595=t,c2596=t,c2597=t,c2598=t,c2599=t,c2600=t,c2601=t,c2602=t,c2603=t,c2604=t,c2605=t," - "c2606=t,c2607=t,c2608=t,c2609=t,c2610=t,c2611=t,c2612=t,c2613=t,c2614=t,c2615=t,c2616=t,c2617=t,c2618=t,c2619=t," - "c2620=t,c2621=t,c2622=t,c2623=t,c2624=t,c2625=t,c2626=t,c2627=t,c2628=t,c2629=t,c2630=t,c2631=t,c2632=t,c2633=t," - "c2634=t,c2635=t,c2636=t,c2637=t,c2638=t,c2639=t,c2640=t,c2641=t,c2642=t,c2643=t,c2644=t,c2645=t,c2646=t,c2647=t," - "c2648=t,c2649=t,c2650=t,c2651=t,c2652=t,c2653=t,c2654=t,c2655=t,c2656=t,c2657=t,c2658=t,c2659=t,c2660=t,c2661=t," - "c2662=t,c2663=t,c2664=t,c2665=t,c2666=t,c2667=t,c2668=t,c2669=t,c2670=t,c2671=t,c2672=t,c2673=t,c2674=t,c2675=t," - "c2676=t,c2677=t,c2678=t,c2679=t,c2680=t,c2681=t,c2682=t,c2683=t,c2684=t,c2685=t,c2686=t,c2687=t,c2688=t,c2689=t," - "c2690=t,c2691=t,c2692=t,c2693=t,c2694=t,c2695=t,c2696=t,c2697=t,c2698=t,c2699=t,c2700=t,c2701=t,c2702=t,c2703=t," - "c2704=t,c2705=t,c2706=t,c2707=t,c2708=t,c2709=t,c2710=t,c2711=t,c2712=t,c2713=t,c2714=t,c2715=t,c2716=t,c2717=t," - "c2718=t,c2719=t,c2720=t,c2721=t,c2722=t,c2723=t,c2724=t," - "c2725=t,c2726=t,c2727=t,c2728=t,c2729=t,c2730=t,c2731=t,c2732=t,c2733=t,c2734=t,c2735=t,c2736=t,c2737=t,c2738=t," - "c2739=t,c2740=t,c2741=t,c2742=t,c2743=t,c2744=t,c2745=t,c2746=t,c2747=t,c2748=t,c2749=t,c2750=t,c2751=t,c2752=t," - "c2753=t,c2754=t,c2755=t,c2756=t,c2757=t,c2758=t,c2759=t,c2760=t,c2761=t,c2762=t,c2763=t,c2764=t,c2765=t,c2766=t," - "c2767=t,c2768=t,c2769=t,c2770=t,c2771=t,c2772=t,c2773=t,c2774=t,c2775=t,c2776=t,c2777=t,c2778=t,c2779=t,c2780=t," - "c2781=t,c2782=t,c2783=t,c2784=t,c2785=t,c2786=t,c2787=t,c2788=t,c2789=t,c2790=t,c2791=t,c2792=t,c2793=t,c2794=t," - "c2795=t,c2796=t,c2797=t,c2798=t,c2799=t,c2800=t,c2801=t,c2802=t,c2803=t,c2804=t,c2805=t,c2806=t,c2807=t,c2808=t," - "c2809=t,c2810=t,c2811=t,c2812=t,c2813=t,c2814=t,c2815=t,c2816=t,c2817=t,c2818=t,c2819=t,c2820=t,c2821=t,c2822=t," - "c2823=t,c2824=t,c2825=t,c2826=t,c2827=t,c2828=t,c2829=t,c2830=t,c2831=t,c2832=t,c2833=t,c2834=t,c2835=t,c2836=t," - "c2837=t,c2838=t,c2839=t,c2840=t,c2841=t,c2842=t,c2843=t,c2844=t,c2845=t,c2846=t,c2847=t,c2848=t,c2849=t,c2850=t," - "c2851=t,c2852=t,c2853=t,c2854=t,c2855=t,c2856=t,c2857=t,c2858=t,c2859=t,c2860=t,c2861=t,c2862=t,c2863=t,c2864=t," - "c2865=t,c2866=t,c2867=t,c2868=t,c2869=t,c2870=t,c2871=t," - "c2872=t,c2873=t,c2874=t,c2875=t,c2876=t,c2877=t,c2878=t,c2879=t,c2880=t,c2881=t,c2882=t,c2883=t,c2884=t,c2885=t," - "c2886=t,c2887=t,c2888=t,c2889=t,c2890=t,c2891=t,c2892=t,c2893=t,c2894=t,c2895=t,c2896=t,c2897=t,c2898=t,c2899=t," - "c2900=t,c2901=t,c2902=t,c2903=t,c2904=t,c2905=t,c2906=t,c2907=t,c2908=t,c2909=t,c2910=t,c2911=t,c2912=t,c2913=t," - "c2914=t,c2915=t,c2916=t,c2917=t,c2918=t,c2919=t,c2920=t,c2921=t,c2922=t,c2923=t,c2924=t,c2925=t,c2926=t,c2927=t," - "c2928=t,c2929=t,c2930=t,c2931=t,c2932=t,c2933=t,c2934=t,c2935=t,c2936=t,c2937=t,c2938=t,c2939=t,c2940=t,c2941=t," - "c2942=t,c2943=t,c2944=t,c2945=t,c2946=t,c2947=t,c2948=t,c2949=t,c2950=t,c2951=t,c2952=t,c2953=t,c2954=t,c2955=t," - "c2956=t,c2957=t,c2958=t,c2959=t,c2960=t,c2961=t,c2962=t,c2963=t,c2964=t,c2965=t,c2966=t,c2967=t,c2968=t,c2969=t," - "c2970=t,c2971=t,c2972=t,c2973=t,c2974=t,c2975=t,c2976=t,c2977=t,c2978=t,c2979=t,c2980=t,c2981=t,c2982=t,c2983=t," - "c2984=t,c2985=t,c2986=t,c2987=t,c2988=t,c2989=t,c2990=t,c2991=t,c2992=t,c2993=t,c2994=t,c2995=t,c2996=t,c2997=t," - "c2998=t,c2999=t,c3000=t,c3001=t,c3002=t,c3003=t,c3004=t,c3005=t,c3006=t,c3007=t,c3008=t,c3009=t,c3010=t,c3011=t," - "c3012=t,c3013=t,c3014=t,c3015=t,c3016=t,c3017=t,c3018=t," - "c3019=t,c3020=t,c3021=t,c3022=t,c3023=t,c3024=t,c3025=t,c3026=t,c3027=t,c3028=t,c3029=t,c3030=t,c3031=t,c3032=t," - "c3033=t,c3034=t,c3035=t,c3036=t,c3037=t,c3038=t,c3039=t,c3040=t,c3041=t,c3042=t,c3043=t,c3044=t,c3045=t,c3046=t," - "c3047=t,c3048=t,c3049=t,c3050=t,c3051=t,c3052=t,c3053=t,c3054=t,c3055=t,c3056=t,c3057=t,c3058=t,c3059=t,c3060=t," - "c3061=t,c3062=t,c3063=t,c3064=t,c3065=t,c3066=t,c3067=t,c3068=t,c3069=t,c3070=t,c3071=t,c3072=t,c3073=t,c3074=t," - "c3075=t,c3076=t,c3077=t,c3078=t,c3079=t,c3080=t,c3081=t,c3082=t,c3083=t,c3084=t,c3085=t,c3086=t,c3087=t,c3088=t," - "c3089=t,c3090=t,c3091=t,c3092=t,c3093=t,c3094=t,c3095=t,c3096=t,c3097=t,c3098=t,c3099=t,c3100=t,c3101=t,c3102=t," - "c3103=t,c3104=t,c3105=t,c3106=t,c3107=t,c3108=t,c3109=t,c3110=t,c3111=t,c3112=t,c3113=t,c3114=t,c3115=t,c3116=t," - "c3117=t,c3118=t,c3119=t,c3120=t,c3121=t,c3122=t,c3123=t,c3124=t,c3125=t,c3126=t,c3127=t,c3128=t,c3129=t,c3130=t," - "c3131=t,c3132=t,c3133=t,c3134=t,c3135=t,c3136=t,c3137=t,c3138=t,c3139=t,c3140=t,c3141=t,c3142=t,c3143=t,c3144=t," - "c3145=t,c3146=t,c3147=t,c3148=t,c3149=t,c3150=t,c3151=t,c3152=t,c3153=t,c3154=t,c3155=t,c3156=t,c3157=t,c3158=t," - "c3159=t,c3160=t,c3161=t,c3162=t,c3163=t,c3164=t,c3165=t," - "c3166=t,c3167=t,c3168=t,c3169=t,c3170=t,c3171=t,c3172=t,c3173=t,c3174=t,c3175=t,c3176=t,c3177=t,c3178=t,c3179=t," - "c3180=t,c3181=t,c3182=t,c3183=t,c3184=t,c3185=t,c3186=t,c3187=t,c3188=t,c3189=t,c3190=t,c3191=t,c3192=t,c3193=t," - "c3194=t,c3195=t,c3196=t,c3197=t,c3198=t,c3199=t,c3200=t,c3201=t,c3202=t,c3203=t,c3204=t,c3205=t,c3206=t,c3207=t," - "c3208=t,c3209=t,c3210=t,c3211=t,c3212=t,c3213=t,c3214=t,c3215=t,c3216=t,c3217=t,c3218=t,c3219=t,c3220=t,c3221=t," - "c3222=t,c3223=t,c3224=t,c3225=t,c3226=t,c3227=t,c3228=t,c3229=t,c3230=t,c3231=t,c3232=t,c3233=t,c3234=t,c3235=t," - "c3236=t,c3237=t,c3238=t,c3239=t,c3240=t,c3241=t,c3242=t,c3243=t,c3244=t,c3245=t,c3246=t,c3247=t,c3248=t,c3249=t," - "c3250=t,c3251=t,c3252=t,c3253=t,c3254=t,c3255=t,c3256=t,c3257=t,c3258=t,c3259=t,c3260=t,c3261=t,c3262=t,c3263=t," - "c3264=t,c3265=t,c3266=t,c3267=t,c3268=t,c3269=t,c3270=t,c3271=t,c3272=t,c3273=t,c3274=t,c3275=t,c3276=t,c3277=t," - "c3278=t,c3279=t,c3280=t,c3281=t,c3282=t,c3283=t,c3284=t,c3285=t,c3286=t,c3287=t,c3288=t,c3289=t,c3290=t,c3291=t," - "c3292=t,c3293=t,c3294=t,c3295=t,c3296=t,c3297=t,c3298=t,c3299=t,c3300=t,c3301=t,c3302=t,c3303=t,c3304=t,c3305=t," - "c3306=t,c3307=t,c3308=t,c3309=t,c3310=t,c3311=t,c3312=t," - "c3313=t,c3314=t,c3315=t,c3316=t,c3317=t,c3318=t,c3319=t,c3320=t,c3321=t,c3322=t,c3323=t,c3324=t,c3325=t,c3326=t," - "c3327=t,c3328=t,c3329=t,c3330=t,c3331=t,c3332=t,c3333=t,c3334=t,c3335=t,c3336=t,c3337=t,c3338=t,c3339=t,c3340=t," - "c3341=t,c3342=t,c3343=t,c3344=t,c3345=t,c3346=t,c3347=t,c3348=t,c3349=t,c3350=t,c3351=t,c3352=t,c3353=t,c3354=t," - "c3355=t,c3356=t,c3357=t,c3358=t,c3359=t,c3360=t,c3361=t,c3362=t,c3363=t,c3364=t,c3365=t,c3366=t,c3367=t,c3368=t," - "c3369=t,c3370=t,c3371=t,c3372=t,c3373=t,c3374=t,c3375=t,c3376=t,c3377=t,c3378=t,c3379=t,c3380=t,c3381=t,c3382=t," - "c3383=t,c3384=t,c3385=t,c3386=t,c3387=t,c3388=t,c3389=t,c3390=t,c3391=t,c3392=t,c3393=t,c3394=t,c3395=t,c3396=t," - "c3397=t,c3398=t,c3399=t,c3400=t,c3401=t,c3402=t,c3403=t,c3404=t,c3405=t,c3406=t,c3407=t,c3408=t,c3409=t,c3410=t," - "c3411=t,c3412=t,c3413=t,c3414=t,c3415=t,c3416=t,c3417=t,c3418=t,c3419=t,c3420=t,c3421=t,c3422=t,c3423=t,c3424=t," - "c3425=t,c3426=t,c3427=t,c3428=t,c3429=t,c3430=t,c3431=t,c3432=t,c3433=t,c3434=t,c3435=t,c3436=t,c3437=t,c3438=t," - "c3439=t,c3440=t,c3441=t,c3442=t,c3443=t,c3444=t,c3445=t,c3446=t,c3447=t,c3448=t,c3449=t,c3450=t,c3451=t,c3452=t," - "c3453=t,c3454=t,c3455=t,c3456=t,c3457=t,c3458=t,c3459=t," - "c3460=t,c3461=t,c3462=t,c3463=t,c3464=t,c3465=t,c3466=t,c3467=t,c3468=t,c3469=t,c3470=t,c3471=t,c3472=t,c3473=t," - "c3474=t,c3475=t,c3476=t,c3477=t,c3478=t,c3479=t,c3480=t,c3481=t,c3482=t,c3483=t,c3484=t,c3485=t,c3486=t,c3487=t," - "c3488=t,c3489=t,c3490=t,c3491=t,c3492=t,c3493=t,c3494=t,c3495=t,c3496=t,c3497=t,c3498=t,c3499=t,c3500=t,c3501=t," - "c3502=t,c3503=t,c3504=t,c3505=t,c3506=t,c3507=t,c3508=t,c3509=t,c3510=t,c3511=t,c3512=t,c3513=t," - "c3514=t,c3515=t,c3516=t,c3517=t,c3518=t,c3519=t,c3520=t,c3521=t,c3522=t,c3523=t,c3524=t,c3525=t,c3526=t,c3527=t," - "c3528=t,c3529=t,c3530=t,c3531=t,c3532=t,c3533=t,c3534=t,c3535=t,c3536=t,c3537=t,c3538=t,c3539=t,c3540=t,c3541=t," - "c3542=t,c3543=t,c3544=t,c3545=t,c3546=t,c3547=t,c3548=t,c3549=t,c3550=t,c3551=t,c3552=t,c3553=t,c3554=t,c3555=t," - "c3556=t,c3557=t,c3558=t,c3559=t,c3560=t,c3561=t,c3562=t,c3563=t,c3564=t,c3565=t,c3566=t,c3567=t,c3568=t,c3569=t," - "c3570=t,c3571=t,c3572=t,c3573=t,c3574=t,c3575=t,c3576=t,c3577=t,c3578=t,c3579=t,c3580=t,c3581=t,c3582=t,c3583=t," - "c3584=t,c3585=t,c3586=t,c3587=t,c3588=t,c3589=t,c3590=t,c3591=t,c3592=t,c3593=t,c3594=t,c3595=t,c3596=t,c3597=t," - "c3598=t,c3599=t,c3600=t,c3601=t,c3602=t,c3603=t,c3604=t,c3605=t,c3606=t,c3607=t,c3608=t,c3609=t,c3610=t,c3611=t," - "c3612=t,c3613=t,c3614=t,c3615=t,c3616=t,c3617=t,c3618=t,c3619=t,c3620=t,c3621=t,c3622=t,c3623=t,c3624=t,c3625=t," - "c3626=t,c3627=t,c3628=t,c3629=t,c3630=t,c3631=t,c3632=t,c3633=t,c3634=t,c3635=t,c3636=t,c3637=t,c3638=t,c3639=t," - "c3640=t,c3641=t,c3642=t,c3643=t,c3644=t,c3645=t,c3646=t,c3647=t,c3648=t,c3649=t,c3650=t,c3651=t,c3652=t,c3653=t," - "c3654=t,c3655=t,c3656=t,c3657=t,c3658=t,c3659=t,c3660=t," - "c3661=t,c3662=t,c3663=t,c3664=t,c3665=t,c3666=t,c3667=t,c3668=t,c3669=t,c3670=t,c3671=t,c3672=t,c3673=t,c3674=t," - "c3675=t,c3676=t,c3677=t,c3678=t,c3679=t,c3680=t,c3681=t,c3682=t,c3683=t,c3684=t,c3685=t,c3686=t,c3687=t,c3688=t," - "c3689=t,c3690=t,c3691=t,c3692=t,c3693=t,c3694=t,c3695=t,c3696=t,c3697=t,c3698=t,c3699=t,c3700=t,c3701=t,c3702=t," - "c3703=t,c3704=t,c3705=t,c3706=t,c3707=t,c3708=t,c3709=t,c3710=t,c3711=t,c3712=t,c3713=t,c3714=t,c3715=t,c3716=t," - "c3717=t,c3718=t,c3719=t,c3720=t,c3721=t,c3722=t,c3723=t,c3724=t,c3725=t,c3726=t,c3727=t,c3728=t,c3729=t,c3730=t," - "c3731=t,c3732=t,c3733=t,c3734=t,c3735=t,c3736=t,c3737=t,c3738=t,c3739=t,c3740=t,c3741=t,c3742=t,c3743=t,c3744=t," - "c3745=t,c3746=t,c3747=t,c3748=t,c3749=t,c3750=t,c3751=t,c3752=t,c3753=t,c3754=t,c3755=t,c3756=t,c3757=t,c3758=t," - "c3759=t,c3760=t,c3761=t,c3762=t,c3763=t,c3764=t,c3765=t,c3766=t,c3767=t,c3768=t,c3769=t,c3770=t,c3771=t,c3772=t," - "c3773=t,c3774=t,c3775=t,c3776=t,c3777=t,c3778=t,c3779=t,c3780=t,c3781=t,c3782=t,c3783=t,c3784=t,c3785=t,c3786=t," - "c3787=t,c3788=t,c3789=t,c3790=t,c3791=t,c3792=t,c3793=t,c3794=t,c3795=t,c3796=t,c3797=t,c3798=t,c3799=t,c3800=t," - "c3801=t,c3802=t,c3803=t,c3804=t,c3805=t,c3806=t,c3807=t," - "c3808=t,c3809=t,c3810=t,c3811=t,c3812=t,c3813=t,c3814=t,c3815=t,c3816=t,c3817=t,c3818=t,c3819=t,c3820=t,c3821=t," - "c3822=t,c3823=t,c3824=t,c3825=t,c3826=t,c3827=t,c3828=t,c3829=t,c3830=t,c3831=t,c3832=t,c3833=t,c3834=t,c3835=t," - "c3836=t,c3837=t,c3838=t,c3839=t,c3840=t,c3841=t,c3842=t,c3843=t,c3844=t,c3845=t,c3846=t,c3847=t,c3848=t,c3849=t," - "c3850=t,c3851=t,c3852=t,c3853=t,c3854=t,c3855=t,c3856=t,c3857=t,c3858=t,c3859=t,c3860=t,c3861=t,c3862=t,c3863=t," - "c3864=t,c3865=t,c3866=t,c3867=t,c3868=t,c3869=t,c3870=t,c3871=t,c3872=t,c3873=t,c3874=t,c3875=t,c3876=t,c3877=t," - "c3878=t,c3879=t,c3880=t,c3881=t,c3882=t,c3883=t,c3884=t,c3885=t,c3886=t,c3887=t,c3888=t,c3889=t,c3890=t,c3891=t," - "c3892=t,c3893=t,c3894=t,c3895=t,c3896=t,c3897=t,c3898=t,c3899=t,c3900=t,c3901=t,c3902=t,c3903=t,c3904=t,c3905=t," - "c3906=t,c3907=t,c3908=t,c3909=t,c3910=t,c3911=t,c3912=t,c3913=t,c3914=t,c3915=t,c3916=t,c3917=t,c3918=t,c3919=t," - "c3920=t,c3921=t,c3922=t,c3923=t,c3924=t,c3925=t,c3926=t,c3927=t,c3928=t,c3929=t,c3930=t,c3931=t,c3932=t,c3933=t," - "c3934=t,c3935=t,c3936=t,c3937=t,c3938=t,c3939=t,c3940=t,c3941=t,c3942=t,c3943=t,c3944=t,c3945=t,c3946=t,c3947=t," - "c3948=t,c3949=t,c3950=t,c3951=t,c3952=t,c3953=t,c3954=t," - "c3955=t,c3956=t,c3957=t,c3958=t,c3959=t,c3960=t,c3961=t,c3962=t,c3963=t,c3964=t,c3965=t,c3966=t,c3967=t,c3968=t," - "c3969=t,c3970=t,c3971=t,c3972=t,c3973=t,c3974=t,c3975=t,c3976=t,c3977=t,c3978=t,c3979=t,c3980=t,c3981=t,c3982=t," - "c3983=t,c3984=t,c3985=t,c3986=t,c3987=t,c3988=t,c3989=t,c3990=t,c3991=t,c3992=t,c3993=t,c3994=t,c3995=t,c3996=t," - "c3997=t,c3998=t,c3999=t,c4000=t,c4001=t,c4002=t,c4003=t,c4004=t,c4005=t,c4006=t,c4007=t,c4008=t,c4009=t,c4010=t," - "c4011=t,c4012=t,c4013=t,c4014=t,c4015=t,c4016=t,c4017=t,c4018=t,c4019=t,c4020=t,c4021=t,c4022=t,c4023=t,c4024=t," - "c4025=t,c4026=t,c4027=t,c4028=t,c4029=t,c4030=t,c4031=t,c4032=t,c4033=t,c4034=t,c4035=t,c4036=t,c4037=t,c4038=t," - "c4039=t,c4040=t,c4041=t,c4042=t,c4043=t,c4044=t,c4045=t,c4046=t,c4047=t,c4048=t,c4049=t,c4050=t,c4051=t,c4052=t," - "c4053=t,c4054=t,c4055=t,c4056=t,c4057=t,c4058=t,c4059=t,c4060=t,c4061=t,c4062=t,c4063=t,c4064=t,c4065=t,c4066=t," - "c4067=t,c4068=t,c4069=t,c4070=t,c4071=t,c4072=t,c4073=t,c4074=t,c4075=t,c4076=t,c4077=t,c4078=t,c4079=t,c4080=t," - "c4081=t,c4082=t,c4083=t,c4084=t,c4085=t,c4086=t,c4087=t,c4088=t,c4089=t,c4090=t,c4091=t,c4092=t,c4093=t " - "1626006833640000000"}; - - int ret = TSDB_CODE_SUCCESS; - for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { - ret = smlParseInfluxLine(info, sql[i], strlen(sql[i])); - if (ret != TSDB_CODE_SUCCESS) break; +//TEST(testCase, smlParseTelnetLine_diff_json_type2_Test) { +// SSmlHandle *info = smlBuildSmlInfo(NULL); +// info->protocol = TSDB_SML_JSON_PROTOCOL; +// ASSERT_NE(info, nullptr); +// +// const char *sql[] = { +// "[{\"metric\":\"sys.cpu.nice\",\"timestamp\": 1346846400,\"value\": 18,\"tags\": {\"host\": \"lga\"}},{\"metric\": \"sys.sdfa\",\"timestamp\": 1346846400,\"value\": \"18\",\"tags\": {\"host\": 8932}},]", +// }; +// for (int i = 0; i < sizeof(sql) / sizeof(sql[0]); i++) { +// char *dataPointStart = (char *)sql[i]; +// int8_t offset[4] = {0}; +// while (1) { +// SSmlLineInfo elements = {0}; +// if(offset[0] == 0){ +// smlJsonParseObjFirst(&dataPointStart, &elements, offset); +// }else{ +// smlJsonParseObj(&dataPointStart, &elements, offset); +// } +// if(*dataPointStart == '\0') break; +// +// SArray *tags = smlJsonParseTags(elements.tags, elements.tags + elements.tagsLen); +// size_t num = taosArrayGetSize(tags); +// ASSERT_EQ(num, 1); +// +// taosArrayDestroy(tags); +// } +// } +// smlDestroyInfo(info); +//} + +TEST(testCase, smlParseNumber_performance_Test) { + char msg[256] = {0}; + SSmlMsgBuf msgBuf; + SSmlKv kv; + + char* str[3] = {"2893f64", "2323u32", "93u8"}; + for (int i = 0; i < 3; ++i) { + int64_t t1 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumber(&kv, &msgBuf); + } + printf("smlParseNumber:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t1); + printf("\n"); + int64_t t2 = taosGetTimestampUs(); + for (int j = 0; j < 10000000; ++j) { + kv.value = str[i]; + kv.length = strlen(str[i]); + smlParseNumberOld(&kv, &msgBuf); + } + printf("smlParseNumberOld:%s cost:%" PRId64, str[i], taosGetTimestampUs() - t2); + printf("\n\n"); } - ASSERT_NE(ret, 0); - smlDestroyInfo(info); -} +} \ No newline at end of file diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 6c86743b696e23ddcdc403860ecafaeb2caa6448..6198f923f4972845784bc57dcf3187c823c0325a 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -120,6 +120,8 @@ static const SSysDbTableSchema userIdxSchema[] = { {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "column_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "index_type", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema userStbsSchema[] = { @@ -178,6 +180,18 @@ static const SSysDbTableSchema userTagsSchema[] = { {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; +static const SSysDbTableSchema userColsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "table_type", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_type", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "col_length", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_precision", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_scale", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "col_nullable", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false} +}; + static const SSysDbTableSchema userTblDistSchema[] = { {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, @@ -218,6 +232,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + // {.name = "compact_start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, }; static const SSysDbTableSchema smaSchema[] = { @@ -294,6 +309,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_STABLES, userStbsSchema, tListLen(userStbsSchema), false}, {TSDB_INS_TABLE_TABLES, userTblsSchema, tListLen(userTblsSchema), false}, {TSDB_INS_TABLE_TAGS, userTagsSchema, tListLen(userTagsSchema), false}, + {TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false}, // {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, {TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), false}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 01a1aa073b2273bc23e35fb2825c5eaf81caf8be..88eb3bdb9745dee95b588e16b7b1ff2ffe3eeabf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -19,7 +19,7 @@ #include "tlog.h" #include "tname.h" -#define MALLOC_ALIGN_BYTES 256 +#define MALLOC_ALIGN_BYTES 32 int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -37,7 +37,8 @@ int32_t colDataGetFullLength(const SColumnInfoData* pColumnInfoData, int32_t num if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length + sizeof(int32_t) * numOfRows; } else { - return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + BitmapLen(numOfRows); + return ((pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) ? 0 : pColumnInfoData->info.bytes * numOfRows) + + BitmapLen(numOfRows); } } @@ -284,7 +285,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->varmeta.allocLen = len + oldLen; } - if (pColumnInfoData->pData && pSource->pData) { // TD-20382 + if (pColumnInfoData->pData && pSource->pData) { // TD-20382 memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); } pColumnInfoData->varmeta.length = len + oldLen; @@ -321,8 +322,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { - if (pColumnInfoData->info.type != pSource->info.type || - (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { + if (pColumnInfoData->info.type != pSource->info.type || (pBlockInfo != NULL && pBlockInfo->capacity < numOfRows)) { return TSDB_CODE_FAILED; } @@ -363,14 +363,14 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSiz size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) { - if (pDataBlock->info.rows > 0) { -// ASSERT(pDataBlock->info.dataLoad == 1); - } - if (pDataBlock == NULL || pDataBlock->info.rows <= 0 || pDataBlock->info.dataLoad == 0) { return 0; } + if (pDataBlock->info.rows > 0) { + // ASSERT(pDataBlock->info.dataLoad == 1); + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); if (numOfCols <= 0) { return -1; @@ -1067,6 +1067,7 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* offset += pInfo->pColData->info.bytes; } + taosMemoryFree(buf); return phelper; } @@ -1200,7 +1201,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case. */ -static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, bool clearPayload) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, + bool clearPayload) { if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } @@ -1259,7 +1261,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* return TSDB_CODE_SUCCESS; } -void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { +void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { @@ -1308,7 +1310,6 @@ void blockDataFreeRes(SSDataBlock* pBlock) { taosArrayDestroy(pBlock->pDataBlock); pBlock->pDataBlock = NULL; taosMemoryFreeClear(pBlock->pBlockAgg); - taosMemoryFree(pBlock->info.pTag); memset(&pBlock->info, 0, sizeof(SDataBlockInfo)); } @@ -1888,7 +1889,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { char pBuf[128] = {0}; int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(dataBlocks, i); + SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; @@ -1900,21 +1901,37 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (k == 0) { + printf("cols:%d |", (int32_t)numOfCols); + } if (colDataIsNull(pColInfoData, rows, j, NULL)) { printf(" %15s |", "NULL"); continue; } + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); printf(" %25s |", pBuf); break; case TSDB_DATA_TYPE_BOOL: - printf(" %15d |", *(int32_t*)var); + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15" PRIi8 " |", *(int8_t*)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15" PRIi16 " |", *(int16_t*)var); break; case TSDB_DATA_TYPE_INT: printf(" %15d |", *(int32_t*)var); break; + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15" PRIu8 " |", *(uint8_t*)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15" PRIu16 " |", *(uint16_t*)var); + break; case TSDB_DATA_TYPE_UINT: printf(" %15u |", *(uint32_t*)var); break; @@ -1964,9 +1981,10 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "\n", + "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, - pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey); + pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, + pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); if (len >= size - 1) return dumpBuf; for (int32_t j = 0; j < rows; j++) { @@ -2068,6 +2086,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param suid * */ +#if 0 int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { int32_t bufSize = sizeof(SSubmitReq); @@ -2231,35 +2250,193 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } +#endif + +int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, const STSchema* pTSchema, + int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + int32_t sz = 1; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + } + + for (int32_t i = 0; i < sz; ++i) { + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + + if (colNum <= 1) { // invalid if only with TS col + continue; + } + + // the rsma result should has the same column number with schema. + ASSERT(colNum == pTSchema->numOfCols); + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; + + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { // iterate by row + + taosArrayClear(pVals); + + bool isStartKey = false; + int32_t offset = 0; + for (int32_t k = 0; k < colNum; ++k) { // iterate by column + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + const STColumn* pCol = &pTSchema->columns[k]; + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + ASSERT(pColInfoData->info.type == pCol->type); + if (!isStartKey) { + isStartKey = true; + ASSERT(PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(TSKEY*)var}); + taosArrayPush(pVals, &cv); + } else if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, (SValue){.val = *(int64_t*)var}); + taosArrayPush(pVals, &cv); + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + ASSERT(0); + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); + } else { + SValue sv; + if (pCol->type == pColInfoData->info.type) { + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + } else { + /** + * 1. sum/avg would convert to int64_t/uint64_t/double during aggregation + * 2. below conversion may lead to overflow or loss, the app should select the right data type. + */ + char tv[8] = {0}; + if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } else { + uint64_t v = 0; + GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); + SET_TYPED_DATA(&tv, pCol->type, v); + } + memcpy(&sv.val, tv, tDataTypes[pCol->type].bytes); + } + SColVal cv = COL_VAL_VALUE(pCol->colId, pColInfoData->info.type, sv); + taosArrayPush(pVals, &cv); + } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + ASSERT(0); + } + break; + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pReq); + } + + return TSDB_CODE_FAILED; + } + *ppReq = pReq; + return TSDB_CODE_SUCCESS; +} char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { if (stbFullName[0] == 0) { return NULL; } - SArray* tags = taosArrayInit(0, sizeof(void*)); + SArray* tags = taosArrayInit(0, sizeof(SSmlKv)); if (tags == NULL) { return NULL; } - SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); - if (pTag == NULL) { - taosArrayDestroy(tags); - return NULL; - } - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); if (cname == NULL) { taosArrayDestroy(tags); - taosMemoryFree(pTag); return NULL; } - pTag->key = "group_id"; - pTag->keyLen = strlen(pTag->key); - pTag->type = TSDB_DATA_TYPE_UBIGINT; - pTag->u = groupId; - pTag->length = sizeof(uint64_t); + SSmlKv pTag = {.key = "group_id", + .keyLen = sizeof("group_id") - 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .u = groupId, + .length = sizeof(uint64_t)}; taosArrayPush(tags, &pTag); RandTableName rname = { @@ -2271,7 +2448,6 @@ char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { buildChildTableName(&rname); - taosMemoryFree(pTag); taosArrayDestroy(tags); if ((rname.ctbShortName && rname.ctbShortName[0]) == 0) { @@ -2350,6 +2526,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) { data += colSizes[col]; colSizes[col] = htonl(colSizes[col]); +// uError("blockEncode col bytes:%d, type:%d, size:%d, htonl size:%d", pColRes->info.bytes, pColRes->info.type, htonl(colSizes[col]), colSizes[col]); } *actualLen = dataLen; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index fc49c033794c946a9881b6ae1efeabcb101e99a7..2bb870837272a0422de500d56e89c7ca245a7beb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,9 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); +static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward); + // SBuffer ================================ void tBufferDestroy(SBuffer *pBuffer) { tFree(pBuffer->pBuf); @@ -61,9 +64,9 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_BIG ((uint8_t)0x30) -#define ROW_BIT_NONE ((uint8_t)0x0) -#define ROW_BIT_NULL ((uint8_t)0x1) -#define ROW_BIT_VALUE ((uint8_t)0x2) +#define BIT_FLG_NONE ((uint8_t)0x0) +#define BIT_FLG_NULL ((uint8_t)0x1) +#define BIT_FLG_VALUE ((uint8_t)0x2) #pragma pack(push, 1) typedef struct { @@ -95,28 +98,29 @@ typedef struct { } \ } while (0) -int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { +int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t code = 0; - ASSERT(taosArrayGetSize(aColVal) > 0); + ASSERT(TARRAY_SIZE(aColVal) > 0); ASSERT(((SColVal *)aColVal->pData)[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(((SColVal *)aColVal->pData)[0].type == TSDB_DATA_TYPE_TIMESTAMP); // scan --------------- - uint8_t flag = 0; - int32_t iColVal = 1; - const int32_t nColVal = taosArrayGetSize(aColVal); - SColVal *pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; - int32_t iTColumn = 1; - STColumn *pTColumn = pTSchema->columns + iTColumn; - int32_t ntp = 0; - int32_t nkv = 0; - int32_t maxIdx = 0; - int32_t nIdx = 0; + SRow *pRow = NULL; + SColVal *colVals = (SColVal *)TARRAY_DATA(aColVal); + uint8_t flag = 0; + int32_t iColVal = 1; + const int32_t nColVal = TARRAY_SIZE(aColVal); + SColVal *pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; + int32_t iTColumn = 1; + const STColumn *pTColumn = pTSchema->columns + iTColumn; + int32_t ntp = 0; + int32_t nkv = 0; + int32_t maxIdx = 0; + int32_t nIdx = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { - ntp += TYPE_BYTES[pTColumn->type]; if (COL_VAL_IS_VALUE(pColVal)) { // VALUE flag |= HAS_VALUE; maxIdx = nkv; @@ -135,21 +139,22 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { nkv += tPutI16v(NULL, -pTColumn->colId); nIdx++; } else { - ASSERT(0); + if (ASSERTS(0, "invalid input")) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; + } } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE flag |= HAS_NONE; - ntp += TYPE_BYTES[pTColumn->type]; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } @@ -161,21 +166,23 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { ntp = sizeof(SRow); break; case HAS_VALUE: - ntp = sizeof(SRow) + ntp; + ntp = sizeof(SRow) + pTSchema->flen + ntp; break; case (HAS_NULL | HAS_NONE): ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1); break; case (HAS_VALUE | HAS_NONE): case (HAS_VALUE | HAS_NULL): - ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT1_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + ntp; + ntp = sizeof(SRow) + BIT2_SIZE(pTSchema->numOfCols - 1) + pTSchema->flen + ntp; break; default: - ASSERT(0); - break; + if (ASSERTS(0, "impossible")) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; + } } if (maxIdx <= UINT8_MAX) { nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; @@ -196,12 +203,14 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } // alloc -------------- - SRow *pRow = NULL; - code = tBufferReserve(pBuffer, nRow, (void **)&pRow); - if (code) return code; + pRow = taosMemoryMalloc(nRow); + if (NULL == pRow) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } // build -------------- - pColVal = (SColVal *)taosArrayGet(aColVal, 0); + pColVal = &colVals[0]; pRow->flag = flag; pRow->rsv = 0; @@ -214,10 +223,10 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } iColVal = 1; - pColVal = (iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; - if (flag & 0xf0) { // KV + if (flag >> 4) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; int32_t iIdx = 0; int32_t nv = 0; @@ -266,11 +275,11 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -302,8 +311,18 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { pv = pf + pTSchema->flen; break; default: - ASSERT(0); - break; + if (ASSERTS(0, "impossible")) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; + } + } + + if (pb) { + if (flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + memset(pb, 0, BIT2_SIZE(pTSchema->numOfCols - 1)); + } else { + memset(pb, 0, BIT1_SIZE(pTSchema->numOfCols - 1)); + } } // build impl @@ -311,7 +330,7 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { if (COL_VAL_IS_VALUE(pColVal)) { // VALUE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_VALUE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_VALUE); if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; @@ -324,24 +343,24 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } else { // NULL - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NULL); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NULL); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { - pColVal = (++iColVal < nColVal) ? (SColVal *)taosArrayGet(aColVal, iColVal) : NULL; + pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE - ROW_SET_BITMAP(pb, flag, iTColumn - 1, ROW_BIT_NONE); + ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } @@ -349,10 +368,16 @@ int32_t tRowBuild(SArray *aColVal, STSchema *pTSchema, SBuffer *pBuffer) { } _exit: + if (code) { + *ppRow = NULL; + tRowDestroy(pRow); + } else { + *ppRow = pRow; + } return code; } -void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { +int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { ASSERT(iCol < pTSchema->numOfCols); ASSERT(pRow->sver == pTSchema->version); @@ -363,20 +388,20 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { pColVal->type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; memcpy(&pColVal->value.val, &pRow->ts, sizeof(TSKEY)); - return; + return 0; } if (pRow->flag == HAS_NONE) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; + return 0; } if (pRow->flag == HAS_NULL) { *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; + return 0; } - if (pRow->flag & 0xf0) { // KV Row + if (pRow->flag >> 4) { // KV Row SKVIdx *pIdx = (SKVIdx *)pRow->data; uint8_t *pv = NULL; if (pRow->flag & KV_FLG_LIT) { @@ -422,7 +447,7 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { memcpy(&pColVal->value.val, pData, pTColumn->bytes); } } - return; + return 0; } else if (TABS(cid) < pTColumn->colId) { lidx = mid + 1; } else { @@ -432,62 +457,190 @@ void tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - uint8_t *pf = NULL; - uint8_t *pv = NULL; - uint8_t bv = ROW_BIT_VALUE; + if (pRow->flag == HAS_VALUE) { + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } + } else { + memcpy(&pColVal->value.val, pRow->data + pTColumn->offset, TYPE_BYTES[pTColumn->type]); + } + } else { + uint8_t *pf; + uint8_t *pv; + uint8_t bv = BIT_FLG_VALUE; - switch (pRow->flag) { - case HAS_VALUE: - pf = pRow->data; - pv = pf + pTSchema->flen; - break; - case (HAS_NULL | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - break; - case (HAS_VALUE | HAS_NONE): - bv = GET_BIT1(pRow->data, iCol - 1); - if (bv) bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL): - bv = GET_BIT1(pRow->data, iCol - 1); - bv++; - pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - case (HAS_VALUE | HAS_NULL | HAS_NONE): - bv = GET_BIT2(pRow->data, iCol - 1); - pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); - pv = pf + pTSchema->flen; - break; - default: - break; - } + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pRow->data, iCol - 1); + if (bv) bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pRow->data, iCol - 1); + bv++; + pf = pRow->data + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pRow->data, iCol - 1); + pf = pRow->data + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERTS(0, "invalid row format"); + return TSDB_CODE_IVLD_DATA_FMT; + } - if (bv == ROW_BIT_NONE) { - *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); - return; - } else if (bv == ROW_BIT_NULL) { - *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); - return; - } + if (bv == BIT_FLG_NONE) { + *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); + return 0; + } else if (bv == BIT_FLG_NULL) { + *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); + return 0; + } - pColVal->cid = pTColumn->colId; - pColVal->type = pTColumn->type; - pColVal->flag = CV_FLAG_VALUE; - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); - if (pColVal->value.nData) { - pColVal->value.pData = pData; + pColVal->cid = pTColumn->colId; + pColVal->type = pTColumn->type; + pColVal->flag = CV_FLAG_VALUE; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + pData += tGetU32v(pData, &pColVal->value.nData); + if (pColVal->value.nData) { + pColVal->value.pData = pData; + } else { + pColVal->value.pData = NULL; + } } else { - pColVal->value.pData = NULL; + memcpy(&pColVal->value.val, pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } - } else { - memcpy(&pColVal->value.val, pv + pTColumn->offset, pTColumn->bytes); } } + + return 0; +} + +void tRowDestroy(SRow *pRow) { + if (pRow) taosMemoryFree(pRow); +} + +static int32_t tRowPCmprFn(const void *p1, const void *p2) { + if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) { + return -1; + } else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) { + return 1; + } + + return 0; +} +static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } +static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { + int32_t code = 0; + + int32_t nRow = iEnd - iStart; + SRowIter **aIter = NULL; + SArray *aColVal = NULL; + SRow *pRow = NULL; + + aIter = taosMemoryCalloc(nRow, sizeof(SRowIter *)); + if (aIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nRow; i++) { + SRow *pRowT = taosArrayGetP(aRowP, iStart + i); + + code = tRowIterOpen(pRowT, pTSchema, &aIter[i]); + if (code) goto _exit; + } + + // merge + aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); + if (aColVal == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iCol = 0; iCol < pTSchema->numOfCols; iCol++) { + SColVal *pColVal = NULL; + for (int32_t iRow = 0; iRow < nRow; iRow++) { + SColVal *pColValT = tRowIterNext(aIter[iRow]); + + // todo: take strategy according to the flag + if (COL_VAL_IS_VALUE(pColValT)) { + pColVal = pColValT; + } else if (COL_VAL_IS_NULL(pColValT)) { + if (pColVal == NULL) { + pColVal = pColValT; + } + } + } + + if (pColVal) taosArrayPush(aColVal, pColVal); + } + + // build + code = tRowBuild(aColVal, pTSchema, &pRow); + if (code) goto _exit; + + taosArrayRemoveBatch(aRowP, iStart, nRow, (FDelete)tRowPDestroy); + taosArrayInsert(aRowP, iStart, &pRow); + +_exit: + if (aIter) { + for (int32_t i = 0; i < nRow; i++) { + tRowIterClose(&aIter[i]); + } + taosMemoryFree(aIter); + } + if (aColVal) taosArrayDestroy(aColVal); + if (code) tRowDestroy(pRow); + return code; +} + +void tRowSort(SArray *aRowP) { + if (TARRAY_SIZE(aRowP) <= 1) return; + taosArraySort(aRowP, tRowPCmprFn); +} + +int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { + int32_t code = 0; + + int32_t iStart = 0; + while (iStart < aRowP->size) { + SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart); + + int32_t iEnd = iStart + 1; + while (iEnd < aRowP->size) { + SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd); + + if (pRow->ts != pRowT->ts) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + code = tRowMergeImpl(aRowP, pTSchema, iStart, iEnd, flag); + if (code) return code; + } + + // the array is also changing, so the iStart just ++ instead of iEnd + iStart++; + } + + return code; } // SRowIter ======================================== @@ -505,10 +658,9 @@ struct SRowIter { uint8_t *pb; uint8_t *pf; }; - uint8_t *pv; }; - - SColVal cv; + uint8_t *pv; + SColVal cv; }; int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { @@ -528,15 +680,15 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; - if (pRow->flag & 0xf0) { + if (pRow->flag >> 4) { pIter->iCol = 0; pIter->pIdx = (SKVIdx *)pRow->data; if (pRow->flag & KV_FLG_LIT) { pIter->pv = pIter->pIdx->idx + pIter->pIdx->nCol; } else if (pRow->flag & KV_FLG_MID) { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 1); // * sizeof(uint16_t) } else { - pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); + pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) } } else { switch (pRow->flag) { @@ -567,7 +719,6 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { _exit: if (code) { *ppIter = NULL; - if (pIter) taosMemoryFree(pIter); } else { *ppIter = pIter; } @@ -608,7 +759,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag & 0xf0) { // KV + if (pIter->pRow->flag >> 4) { // KV if (pIter->iCol < pIter->pIdx->nCol) { uint8_t *pData; @@ -656,7 +807,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } } else { // Tuple - uint8_t bv = ROW_BIT_VALUE; + uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { switch (pIter->pRow->flag) { case (HAS_NULL | HAS_NONE): @@ -677,10 +828,10 @@ SColVal *tRowIterNext(SRowIter *pIter) { break; } - if (bv == ROW_BIT_NONE) { + if (bv == BIT_FLG_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; - } else if (bv == ROW_BIT_NULL) { + } else if (bv == BIT_FLG_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -698,7 +849,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.value.pData = NULL; } } else { - memcpy(&pIter->cv.value.val, pIter->pv + pTColumn->offset, pTColumn->bytes); + memcpy(&pIter->cv.value.val, pIter->pf + pTColumn->offset, TYPE_BYTES[pTColumn->type]); } goto _exit; } @@ -708,7 +859,276 @@ _exit: return &pIter->cv; } -// STSchema ======================================== +static int32_t tRowNoneUpsertColData(SColData *aColData, int32_t nColData, int32_t flag) { + int32_t code = 0; + + if (flag) return code; + + for (int32_t iColData = 0; iColData < nColData; iColData++) { + code = tColDataAppendValueImpl[aColData[iColData].flag][CV_FLAG_NONE](&aColData[iColData], NULL, 0); + if (code) return code; + } + + return code; +} +static int32_t tRowNullUpsertColData(SColData *aColData, int32_t nColData, STSchema *pSchema, int32_t flag) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { // NULL + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0); + } + if (code) goto _exit; + + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } + } else { // NONE + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, + int32_t flag) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + + uint8_t *pb = NULL, *pf = NULL, *pv = NULL; + + switch (pRow->flag) { + case HAS_VALUE: + pf = pRow->data; + pv = pf + pTSchema->flen; + break; + case (HAS_NULL | HAS_NONE): + pb = pRow->data; + break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pb = pRow->data; + pf = pb + BIT1_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pb = pRow->data; + pf = pb + BIT2_SIZE(pTSchema->numOfCols - 1); + pv = pf + pTSchema->flen; + break; + default: + ASSERTS(0, "Invalid row flag"); + return TSDB_CODE_IVLD_DATA_FMT; + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + ASSERT(pTColumn->type == pColData->type); + if (pb) { + uint8_t bv; + switch (pRow->flag) { + case (HAS_NULL | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + break; + case (HAS_VALUE | HAS_NONE): + bv = GET_BIT1(pb, iTColumn - 1); + if (bv) bv++; + break; + case (HAS_VALUE | HAS_NULL): + bv = GET_BIT1(pb, iTColumn - 1) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + bv = GET_BIT2(pb, iTColumn - 1); + break; + default: + ASSERTS(0, "Invalid row flag"); + return TSDB_CODE_IVLD_DATA_FMT; + } + + if (bv == BIT_FLG_NONE) { + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) + goto _exit; + goto _continue; + } else if (bv == BIT_FLG_NULL) { + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0); + } + if (code) goto _exit; + goto _continue; + } + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { + uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); + uint32_t nData; + pData += tGetU32v(pData, &nData); + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0); + } + if (code) goto _exit; + } else { + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset, + TYPE_BYTES[pColData->type]); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pf + pTColumn->offset, + TYPE_BYTES[pColData->type], flag > 0); + } + if (code) goto _exit; + } + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) { + int32_t code = 0; + + SKVIdx *pKVIdx = (SKVIdx *)pRow->data; + uint8_t *pv = NULL; + int32_t iColData = 0; + SColData *pColData = &aColData[iColData]; + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + int32_t iCol = 0; + + if (pRow->flag & KV_FLG_LIT) { + pv = pKVIdx->idx + pKVIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pKVIdx->idx + (pKVIdx->nCol << 1); + } else if (pRow->flag & KV_FLG_BIG) { + pv = pKVIdx->idx + (pKVIdx->nCol << 2); + } else { + ASSERT(0); + } + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + while (iCol < pKVIdx->nCol) { + uint8_t *pData; + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pKVIdx->idx)[iCol]; + } else if (pRow->flag & KV_FLG_BIG) { + pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; + } else { + ASSERTS(0, "Invalid KV row format"); + return TSDB_CODE_IVLD_DATA_FMT; + } + + int16_t cid; + pData += tGetI16v(pData, &cid); + + if (TABS(cid) == pTColumn->colId) { + if (cid < 0) { + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0, flag > 0); + } + if (code) goto _exit; + } else { + uint32_t nData; + if (IS_VAR_DATA_TYPE(pTColumn->type)) { + pData += tGetU32v(pData, &nData); + } else { + nData = 0; + } + if (flag == 0) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); + } else { + code = tColDataUpdateValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData, flag > 0); + } + if (code) goto _exit; + } + iCol++; + goto _continue; + } else if (TABS(cid) > pTColumn->colId) { // NONE + break; + } else { + iCol++; + } + } + + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + + _continue: + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + if (flag == 0 && (code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0))) goto _exit; + pColData = (++iColData < nColData) ? &aColData[iColData] : NULL; + } + } + +_exit: + return code; +} +/* flag > 0: forward update + * flag == 0: append + * flag < 0: backward update + */ +int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag) { + ASSERT(pRow->sver == pTSchema->version); + ASSERT(nColData > 0); + + if (pRow->flag == HAS_NONE) { + return tRowNoneUpsertColData(aColData, nColData, flag); + } else if (pRow->flag == HAS_NULL) { + return tRowNullUpsertColData(aColData, nColData, pTSchema, flag); + } else if (pRow->flag >> 4) { // KV row + return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag); + } else { // TUPLE row + return tRowTupleUpsertColData(pRow, pTSchema, aColData, nColData, flag); + } +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1086,91 +1506,7 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { tPutI16v(p + offset, cid); } -#if 1 // =================================================================================================================== -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - if (pBuilder == NULL) return -1; - - pBuilder->tCols = 256; - pBuilder->columns = (STColumn *)taosMemoryMalloc(sizeof(STColumn) * pBuilder->tCols); - if (pBuilder->columns == NULL) return -1; - - tdResetTSchemaBuilder(pBuilder, version); - return 0; -} - -void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder) { - taosMemoryFreeClear(pBuilder->columns); - } -} - -void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { - pBuilder->nCols = 0; - pBuilder->tlen = 0; - pBuilder->flen = 0; - pBuilder->version = version; -} - -int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int8_t flags, col_id_t colId, col_bytes_t bytes) { - if (!isValidDataType(type)) return -1; - - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - STColumn *columns = (STColumn *)taosMemoryRealloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); - if (columns == NULL) return -1; - pBuilder->columns = columns; - } - - STColumn *pCol = &(pBuilder->columns[pBuilder->nCols]); - pCol->type = type; - pCol->colId = colId; - pCol->flags = flags; - if (pBuilder->nCols == 0) { - pCol->offset = -1; - } else { - pCol->offset = pBuilder->flen; - pBuilder->flen += TYPE_BYTES[type]; - } - - if (IS_VAR_DATA_TYPE(type)) { - pCol->bytes = bytes; - pBuilder->tlen += (TYPE_BYTES[type] + bytes); - } else { - pCol->bytes = TYPE_BYTES[type]; - pBuilder->tlen += TYPE_BYTES[type]; - } - - pBuilder->nCols++; - - ASSERT(pCol->offset < pBuilder->flen); - - return 0; -} - -STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { - if (pBuilder->nCols <= 0) return NULL; - - int tlen = sizeof(STSchema) + sizeof(STColumn) * pBuilder->nCols; - - STSchema *pSchema = (STSchema *)taosMemoryMalloc(tlen); - if (pSchema == NULL) return NULL; - - pSchema->version = pBuilder->version; - pSchema->numOfCols = pBuilder->nCols; - pSchema->tlen = pBuilder->tlen; - pSchema->flen = pBuilder->flen; - -#ifdef TD_SUPPORT_BITMAP - pSchema->tlen += (int)TD_BITMAP_BYTES(pSchema->numOfCols); -#endif - - memcpy(&pSchema->columns[0], pBuilder->columns, sizeof(STColumn) * pBuilder->nCols); - - return pSchema; -} - -#endif - +// STSchema ======================================== STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); if (pTSchema == NULL) return NULL; @@ -1184,7 +1520,7 @@ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { pTSchema->columns[0].colId = aSchema[0].colId; pTSchema->columns[0].type = aSchema[0].type; pTSchema->columns[0].flags = aSchema[0].flags; - pTSchema->columns[0].bytes = aSchema[0].bytes; + pTSchema->columns[0].bytes = TYPE_BYTES[aSchema[0].type]; pTSchema->columns[0].offset = -1; // other columns @@ -1195,17 +1531,24 @@ STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { pTColumn->colId = pSchema->colId; pTColumn->type = pSchema->type; pTColumn->flags = pSchema->flags; - pTColumn->bytes = pSchema->bytes; pTColumn->offset = pTSchema->flen; - pTSchema->flen += TYPE_BYTES[pTColumn->type]; + if (IS_VAR_DATA_TYPE(pSchema->type)) { + pTColumn->bytes = pSchema->bytes; + pTSchema->tlen += (TYPE_BYTES[pSchema->type] + pSchema->bytes); // todo: remove + } else { + pTColumn->bytes = TYPE_BYTES[pSchema->type]; + pTSchema->tlen += TYPE_BYTES[pSchema->type]; // todo: remove + } + + pTSchema->flen += TYPE_BYTES[pTColumn->type]; } - return pTSchema; -} +#if 1 // todo : remove this + pTSchema->tlen += (int32_t)TD_BITMAP_BYTES(numOfCols); +#endif -void tDestroyTSchema(STSchema *pTSchema) { - if (pTSchema) taosMemoryFree(pTSchema); + return pTSchema; } // SColData ======================================== @@ -1213,7 +1556,7 @@ void tColDataDestroy(void *ph) { SColData *pColData = (SColData *)ph; tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); + tFree(pColData->aOffset); tFree(pColData->pData); } @@ -1225,30 +1568,45 @@ void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { } void tColDataClear(SColData *pColData) { + pColData->numOfNone = 0; + pColData->numOfNull = 0; + pColData->numOfValue = 0; pColData->nVal = 0; pColData->flag = 0; pColData->nData = 0; } -static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { +void tColDataDeepClear(SColData *pColData) { + pColData->pBitMap = NULL; + pColData->aOffset = NULL; + pColData->pData = NULL; + + tColDataClear(pColData); +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColData->type)) { - code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + code = tRealloc((uint8_t **)(&pColData->aOffset), ((int64_t)(pColData->nVal + 1)) << 2); if (code) goto _exit; pColData->aOffset[pColData->nVal] = pColData->nData; - if (pColVal->value.nData) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (nData) { + code = tRealloc(&pColData->pData, pColData->nData + nData); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; + memcpy(pColData->pData + pColData->nData, pData, nData); + pColData->nData += nData; } } else { ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, &pColVal->value.val, tDataTypes[pColData->type].bytes); + if (pData) { + memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + } else { + memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); + } pColData->nData += tDataTypes[pColData->type].bytes; } pColData->nVal++; @@ -1256,21 +1614,24 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_VALUE; - return tColDataPutValue(pColData, pColVal); + pColData->numOfValue++; + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NONE; + pColData->numOfNone++; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->flag = HAS_NULL; + pColData->numOfNull++; pColData->nVal++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1278,9 +1639,10 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; + pColData->numOfValue++; if (pColData->nVal) { if (IS_VAR_DATA_TYPE(pColData->type)) { @@ -1296,13 +1658,14 @@ static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; + pColData->numOfNone++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1310,14 +1673,15 @@ static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_NULL; + pColData->numOfNull++; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1325,9 +1689,10 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); pColData->flag |= HAS_VALUE; + pColData->numOfValue++; if (pColData->nVal) { if (IS_VAR_DATA_TYPE(pColData->type)) { @@ -1343,9 +1708,9 @@ static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); @@ -1353,30 +1718,33 @@ static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *p if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); pColData->flag |= HAS_NONE; + pColData->numOfNone++; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, uint8_t *pData, uint32_t nData) { pColData->nVal++; + pColData->numOfNull++; return 0; } -static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_VALUE; + pColData->numOfValue++; uint8_t *pBitMap = NULL; code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); } - SET_BIT2(pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pBitMap, pColData->nVal, 2); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; @@ -1395,163 +1763,179 @@ static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *p } } - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); + pColData->numOfNone++; pColData->nVal++; return code; } -static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); + pColData->numOfNull++; pColData->nVal++; return code; } -#define tColDataAppendValue40 tColDataPutValue -static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, uint8_t *pData, uint32_t nData) { + pColData->numOfValue++; + return tColDataPutValue(pColData, pData, nData); +} +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; + pColData->numOfNone++; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit); if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; + pColData->numOfNull++; int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit); if (code) return code; memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); + pColData->numOfValue++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); + pColData->numOfNone++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NULL; + pColData->numOfNull++; uint8_t *pBitMap = NULL; code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pBitMap, pColData->nVal, 1); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 1); + pColData->numOfValue++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; pColData->flag |= HAS_NONE; + pColData->numOfNone++; uint8_t *pBitMap = NULL; code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + SET_BIT2_EX(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pBitMap, pColData->nVal, 0); tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + SET_BIT1_EX(pColData->pBitMap, pColData->nVal, 0); + pColData->numOfNull++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 2); + pColData->numOfValue++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, pData, nData); } -static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 0); + pColData->numOfNone++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + SET_BIT2_EX(pColData->pBitMap, pColData->nVal, 1); + pColData->numOfNull++; - return tColDataPutValue(pColData, pColVal); + return tColDataPutValue(pColData, NULL, 0); } -static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL @@ -1560,10 +1944,325 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pCo {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE + + // VALUE NONE NULL }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag]( + pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, + pColVal->value.nData); +} + +static FORCE_INLINE int32_t tColDataUpdateValue10(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + pColData->numOfNone--; + pColData->nVal--; + if (pColData->numOfNone) { + return tColDataAppendValue10(pColData, pData, nData); + } else { + pColData->flag = 0; + return tColDataAppendValue00(pColData, pData, nData); + } +} +static FORCE_INLINE int32_t tColDataUpdateValue12(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + pColData->numOfNone--; + pColData->nVal--; + if (pColData->numOfNone) { + return tColDataAppendValue12(pColData, pData, nData); + } else { + pColData->flag = 0; + return tColDataAppendValue02(pColData, pData, nData); + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue20(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (forward) { + pColData->numOfNull--; + pColData->nVal--; + if (pColData->numOfNull) { + return tColDataAppendValue20(pColData, pData, nData); + } else { + pColData->flag = 0; + return tColDataAppendValue00(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue30(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> VALUE + pColData->numOfNone--; + pColData->nVal--; + if (pColData->numOfNone) { + return tColDataAppendValue30(pColData, pData, nData); + } else { + pColData->flag = HAS_NULL; + return tColDataAppendValue20(pColData, pData, nData); + } + } else if (forward) { // NULL ==> VALUE + pColData->numOfNull--; + pColData->nVal--; + if (pColData->numOfNull) { + return tColDataAppendValue30(pColData, pData, nData); + } else { + pColData->flag = HAS_NONE; + return tColDataAppendValue10(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue32(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> NULL + pColData->numOfNone--; + pColData->numOfNull++; + if (pColData->numOfNone) { + SET_BIT1(pColData->pBitMap, pColData->nVal - 1, 1); + } else { + pColData->flag = HAS_NULL; + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue40(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (forward) { // VALUE ==> VALUE + pColData->nVal--; + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataPutValue(pColData, pData, nData); + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue42(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (forward) { // VALUE ==> NULL + pColData->numOfValue--; + pColData->nVal--; + if (pColData->numOfValue) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataAppendValue42(pColData, pData, nData); + } else { + pColData->flag = 0; + pColData->nData = 0; + return tColDataAppendValue02(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue50(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> VALUE + pColData->numOfNone--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNone) { + return tColDataAppendValue50(pColData, pData, nData); + } else { + pColData->flag = HAS_VALUE; + return tColDataAppendValue40(pColData, pData, nData); + } + } else if (forward) { // VALUE ==> VALUE + pColData->nVal--; + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataPutValue(pColData, pData, nData); + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue52(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NONE ==> NULL + pColData->numOfNone--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNone) { + return tColDataAppendValue52(pColData, pData, nData); + } else { + pColData->flag = HAS_VALUE; + return tColDataAppendValue42(pColData, pData, nData); + } + } else if (forward) { // VALUE ==> NULL + pColData->numOfValue--; + pColData->nVal--; + if (pColData->numOfValue) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataAppendValue52(pColData, pData, nData); + } else { + pColData->flag = HAS_NONE; + pColData->nData = 0; + return tColDataAppendValue12(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue60(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (forward) { + if (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 0) { // NULL ==> VALUE + pColData->numOfNull--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNull) { + return tColDataAppendValue60(pColData, pData, nData); + } else { + pColData->flag = HAS_VALUE; + return tColDataAppendValue40(pColData, pData, nData); + } + } else { // VALUE ==> VALUE + pColData->nVal--; + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataPutValue(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue62(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + if (forward && (GET_BIT1(pColData->pBitMap, pColData->nVal - 1) == 1)) { // VALUE ==> NULL + pColData->numOfValue--; + pColData->nVal--; + if (pColData->numOfValue) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataAppendValue62(pColData, pData, nData); + } else { + pColData->flag = HAS_NULL; + pColData->nData = 0; + return tColDataAppendValue20(pColData, pData, nData); + } + } + return 0; +} +static FORCE_INLINE int32_t tColDataUpdateValue70(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + int32_t code = 0; + + uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1); + if (bv == 0) { // NONE ==> VALUE + pColData->numOfNone--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNone) { + return tColDataAppendValue70(pColData, pData, nData); + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1); + } + pColData->flag = (HAS_VALUE | HAS_NULL); + return tColDataAppendValue60(pColData, pData, nData); + } + } else if (bv == 1) { // NULL ==> VALUE + if (forward) { + pColData->numOfNull--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNull) { + return tColDataAppendValue70(pColData, pData, nData); + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) ? 1 : 0); + } + pColData->flag = (HAS_VALUE | HAS_NONE); + return tColDataAppendValue50(pColData, pData, nData); + } + } + } else if (bv == 2) { // VALUE ==> VALUE + if (forward) { + pColData->nVal--; + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataPutValue(pColData, pData, nData); + } + } else { + ASSERT(0); + } + return 0; +} +static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) { + uint8_t bv = GET_BIT2(pColData->pBitMap, pColData->nVal - 1); + if (bv == 0) { // NONE ==> NULL + pColData->numOfNone--; + pColData->nVal--; + if (!IS_VAR_DATA_TYPE(pColData->type)) { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + if (pColData->numOfNone) { + return tColDataAppendValue72(pColData, pData, nData); + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal) - 1); + } + pColData->flag = (HAS_VALUE | HAS_NULL); + return tColDataAppendValue62(pColData, pData, nData); + } + } else if (bv == 2 && forward) { // VALUE ==> NULL + pColData->numOfValue--; + pColData->nVal--; + if (pColData->numOfValue) { + if (IS_STR_DATA_TYPE(pColData->type)) { + pColData->nData = pColData->aOffset[pColData->nVal]; + } else { + pColData->nData -= TYPE_BYTES[pColData->type]; + } + return tColDataAppendValue72(pColData, pData, nData); + } else { + for (int32_t iVal = 0; iVal < pColData->nVal; ++iVal) { + SET_BIT1(pColData->pBitMap, iVal, GET_BIT2(pColData->pBitMap, iVal)); + } + pColData->flag = (HAS_NULL | HAS_NONE); + pColData->nData = 0; + return tColDataAppendValue32(pColData, pData, nData); + } + } + return 0; +} +static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward) = { + {NULL, NULL, NULL}, // 0 + {tColDataUpdateValue10, NULL, tColDataUpdateValue12}, // HAS_NONE + {tColDataUpdateValue20, NULL, NULL}, // HAS_NULL + {tColDataUpdateValue30, NULL, tColDataUpdateValue32}, // HAS_NULL|HAS_NONE + {tColDataUpdateValue40, NULL, tColDataUpdateValue42}, // HAS_VALUE + {tColDataUpdateValue50, NULL, tColDataUpdateValue52}, // HAS_VALUE|HAS_NONE + {tColDataUpdateValue60, NULL, tColDataUpdateValue62}, // HAS_VALUE|HAS_NULL + {tColDataUpdateValue70, NULL, tColDataUpdateValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE + + // VALUE NONE NULL +}; +int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward) { + ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); + ASSERT(pColData->nVal > 0); + + if (tColDataUpdateValueImpl[pColData->flag][pColVal->flag] == NULL) return 0; + + return tColDataUpdateValueImpl[pColData->flag][pColVal->flag]( + pColData, IS_VAR_DATA_TYPE(pColData->type) ? pColVal->value.pData : (uint8_t *)&pColVal->value.val, + pColVal->value.nData, forward); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE @@ -1572,7 +2271,8 @@ static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SCo static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); } -static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_NULL|HAS_NONE switch (GET_BIT1(pColData->pBitMap, iVal)) { case 0: *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); @@ -1656,75 +2356,795 @@ void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { } uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal) { - uint8_t v; switch (pColData->flag) { case HAS_NONE: - v = 0; - break; + return 0; case HAS_NULL: - v = 1; - break; + return 1; case (HAS_NULL | HAS_NONE): - v = GET_BIT1(pColData->pBitMap, iVal); - break; + return GET_BIT1(pColData->pBitMap, iVal); case HAS_VALUE: - v = 2; - break; + return 2; case (HAS_VALUE | HAS_NONE): - v = GET_BIT1(pColData->pBitMap, iVal); - if (v) v = 2; - break; + return (GET_BIT1(pColData->pBitMap, iVal)) ? 2 : 0; case (HAS_VALUE | HAS_NULL): - v = GET_BIT1(pColData->pBitMap, iVal) + 1; + return GET_BIT1(pColData->pBitMap, iVal) + 1; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + return GET_BIT2(pColData->pBitMap, iVal); + default: + ASSERTS(0, "not possible"); + return 0; + } +} + +int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg) { + int32_t code = 0; + + *pColData = *pColDataFrom; + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = xMalloc(arg, BIT1_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT1_SIZE(pColData->nVal)); break; case (HAS_VALUE | HAS_NULL | HAS_NONE): - v = GET_BIT2(pColData->pBitMap, iVal); + pColData->pBitMap = xMalloc(arg, BIT2_SIZE(pColData->nVal)); + if (pColData->pBitMap == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->pBitMap, pColDataFrom->pBitMap, BIT2_SIZE(pColData->nVal)); break; default: - ASSERT(0); + pColData->pBitMap = NULL; break; } - return v; + + // offset + if (IS_VAR_DATA_TYPE(pColData->type) && (pColData->flag & HAS_VALUE)) { + pColData->aOffset = xMalloc(arg, pColData->nVal << 2); + if (pColData->aOffset == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pColData->aOffset, pColDataFrom->aOffset, pColData->nVal << 2); + } else { + pColData->aOffset = NULL; + } + + // value + if (pColData->nData) { + pColData->pData = xMalloc(arg, pColData->nData); + if (pColData->pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + memcpy(pColData->pData, pColDataFrom->pData, pColData->nData); + } else { + pColData->pData = NULL; + } + +_exit: + return code; +} + +int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, + char *data) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(type)) { // var-length data type + for (int32_t i = 0; i < nRows; ++i) { + int32_t offset = *((int32_t *)lengthOrbitmap + i); + if (offset == -1) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), + varDataLen(data + offset)); + } + } + } else { // fixed-length data type + bool allValue = true; + bool allNull = true; + for (int32_t i = 0; i < nRows; ++i) { + if (!colDataIsNull_f(lengthOrbitmap, i)) { + allNull = false; + } else { + allValue = false; + } + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < nRows; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < nRows; ++i) { + if (colDataIsNull_f(lengthOrbitmap, i)) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)data + bytes * i, bytes); + } + } + } + } + +_exit: + return code; +} + +int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { + int32_t code = 0; + + ASSERT(pColData->type == pBind->buffer_type); + + if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null && pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + pBind->buffer_length * i, pBind->length[i]); + } + } + } else { // fixed-length data type + bool allValue; + bool allNull; + if (pBind->is_null) { + bool same = (memcmp(pBind->is_null, pBind->is_null + 1, pBind->num - 1) == 0); + allNull = (same && pBind->is_null[0] != 0); + allValue = (same && pBind->is_null[0] == 0); + } else { + allNull = false; + allValue = true; + } + + if (allValue) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } else if (allNull) { + // optimize (todo) + for (int32_t i = 0; i < pBind->num; ++i) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } + } else { + for (int32_t i = 0; i < pBind->num; ++i) { + if (pBind->is_null[i]) { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE]( + pColData, (uint8_t *)pBind->buffer + TYPE_BYTES[pColData->type] * i, pBind->buffer_length); + } + } + } + } + +_exit: + return code; } -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { +static int32_t tColDataSwapValue(SColData *pColData, int32_t i, int32_t j) { int32_t code = 0; - int32_t size; - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid == pColDataSrc->cid); - ASSERT(pColDataDest->type == pColDataSrc->type); + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nData1 = pColData->aOffset[i + 1] - pColData->aOffset[i]; + int32_t nData2 = (j < pColData->nVal - 1) ? pColData->aOffset[j + 1] - pColData->aOffset[j] + : pColData->nData - pColData->aOffset[j]; + uint8_t *pData = taosMemoryMalloc(TMAX(nData1, nData2)); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; + if (nData1 > nData2) { + memcpy(pData, pColData->pData + pColData->aOffset[i], nData1); + memcpy(pColData->pData + pColData->aOffset[i], pColData->pData + pColData->aOffset[j], nData2); + // memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i] + nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pData, nData1); + } else { + memcpy(pData, pColData->pData + pColData->aOffset[j], nData2); + memcpy(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i], nData1); + // memmove(pColData->pData + pColData->aOffset[j] + nData2 - nData1, pColData->pData + pColData->aOffset[i] + + // nData1, + // pColData->aOffset[j] - pColData->aOffset[i + 1]); + memmove(pColData->pData + pColData->aOffset[i] + nData2, pColData->pData + pColData->aOffset[i + 1], + pColData->aOffset[j] - pColData->aOffset[i + 1]); + memcpy(pColData->pData + pColData->aOffset[i], pData, nData2); + } + for (int32_t k = i + 1; k <= j; ++k) { + pColData->aOffset[k] = pColData->aOffset[k] + nData2 - nData1; + } + + taosMemoryFree(pData); + } else { + uint64_t val; + memcpy(&val, &pColData->pData[TYPE_BYTES[pColData->type] * i], TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * i], &pColData->pData[TYPE_BYTES[pColData->type] * j], + TYPE_BYTES[pColData->type]); + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * j], &val, TYPE_BYTES[pColData->type]); + } + +_exit: + return code; +} + +static void tColDataSwap(SColData *pColData, int32_t i, int32_t j) { + ASSERT(i < j); + ASSERT(j < pColData->nVal); + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + } break; + case HAS_VALUE: { + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + SET_BIT1(pColData->pBitMap, i, GET_BIT1(pColData->pBitMap, j)); + SET_BIT1(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv = GET_BIT2(pColData->pBitMap, i); + SET_BIT2(pColData->pBitMap, i, GET_BIT2(pColData->pBitMap, j)); + SET_BIT2(pColData->pBitMap, j, bv); + tColDataSwapValue(pColData, i, j); + } break; + default: + ASSERT(0); + break; + } +} + +static int32_t tColDataCopyRowCell(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + if (IS_VAR_DATA_TYPE(pToColData->type)) { + int32_t nData = (iFromRow < pFromColData->nVal - 1) + ? pFromColData->aOffset[iFromRow + 1] - pFromColData->aOffset[iFromRow] + : pFromColData->nData - pFromColData->aOffset[iFromRow]; + if (iToRow == 0) { + pToColData->aOffset[iToRow] = 0; + } + + if (iToRow < pToColData->nVal - 1) { + pToColData->aOffset[iToRow + 1] = pToColData->aOffset[iToRow] + nData; + } + + memcpy(pToColData->pData + pToColData->aOffset[iToRow], pFromColData->pData + pFromColData->aOffset[iFromRow], + nData); + } else { + memcpy(&pToColData->pData[TYPE_BYTES[pToColData->type] * iToRow], + &pFromColData->pData[TYPE_BYTES[pToColData->type] * iFromRow], TYPE_BYTES[pToColData->type]); + } + return code; +} + +static int32_t tColDataCopyRowSingleCol(SColData *pFromColData, int32_t iFromRow, SColData *pToColData, + int32_t iToRow) { + int32_t code = TSDB_CODE_SUCCESS; + + switch (pFromColData->flag) { + case HAS_NONE: + case HAS_NULL: + break; + case (HAS_NULL | HAS_NONE): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + } break; + case HAS_VALUE: { + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): { + SET_BIT1(pToColData->pBitMap, iToRow, GET_BIT1(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + SET_BIT2(pToColData->pBitMap, iToRow, GET_BIT2(pFromColData->pBitMap, iFromRow)); + tColDataCopyRowCell(pFromColData, iFromRow, pToColData, iToRow); + } break; + default: + return -1; + } + + return code; +} + +static int32_t tColDataCopyRow(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t iToRow, + int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + code = tColDataCopyRowSingleCol(&aFromColData[i], iFromRow, &aToColData[i], iToRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataCopyRowAppend(SColData *aFromColData, int32_t iFromRow, SColData *aToColData, int32_t nColData) { + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < nColData; i++) { + SColVal cv = {0}; + tColDataGetValue(&aFromColData[i], iFromRow, &cv); + code = tColDataAppendValue(&aToColData[i], &cv); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + return code; +} + +static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t mid, int32_t end, int32_t nColData) { + SColData *aDstColData = NULL; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + + int32_t i = start, j = mid + 1, k = 0; + + if (end > start) { + aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); + for (int c = 0; c < nColData; ++c) { + tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); + } + if (aDstColData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + /* + for (int32_t i = 0; i < nColData; i++) { + tColDataCopy(&aColData[i], &aDstColData[i], tColDataDefaultMalloc, NULL); + } + */ + } + + while (i <= mid && j <= end) { + if (aKey[i] <= aKey[j]) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } else { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + } + + while (i <= mid) { + // tColDataCopyRow(aColData, i++, aDstColData, k++); + tColDataCopyRowAppend(aColData, i++, aDstColData, nColData); + } + + while (j <= end) { + // tColDataCopyRow(aColData, j++, aDstColData, k++); + tColDataCopyRowAppend(aColData, j++, aDstColData, nColData); + } + + for (i = start, k = 0; i <= end; ++i, ++k) { + tColDataCopyRow(aDstColData, k, aColData, i, nColData); + } + + if (aDstColData) { + for (int32_t i = 0; i < nColData; i++) { + tColDataDestroy(&aDstColData[i]); + } + taosMemoryFree(aDstColData); + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t tColDataMergeSort(SColData *aColData, int32_t start, int32_t end, int32_t nColData) { + int32_t ret = TSDB_CODE_SUCCESS; + int32_t mid; + + if (start >= end) { + return TSDB_CODE_SUCCESS; + } + + mid = (start + end) / 2; + + ret = tColDataMergeSort(aColData, start, mid, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + ret = tColDataMergeSort(aColData, mid + 1, end, nColData); + if (ret != TSDB_CODE_SUCCESS) { + return ret; + } + + return tColDataMergeSortMerge(aColData, start, mid, end, nColData); +} + +static int32_t tColDataSort(SColData *aColData, int32_t nColData) { + int32_t nVal = aColData[0].nVal; + + if (nVal < 2) return TSDB_CODE_SUCCESS; + + return tColDataMergeSort(aColData, 0, nVal - 1, nColData); +} +static void tColDataMergeImpl(SColData *pColData, int32_t iStart, int32_t iEnd /* not included */) { + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: { + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_NULL | HAS_NONE): { + if (GET_BIT1(pColData->pBitMap, iStart) == 0) { + for (int32_t i = iStart + 1; i < iEnd; ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 1) { + SET_BIT1(pColData->pBitMap, iStart, 1); + break; + } + } + } + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + uint8_t bv = GET_BIT1(pColData->pBitMap, i); + if (bv == BIT_FLG_NONE) { + flag |= HAS_NONE; + } else if (bv == BIT_FLG_NULL) { + flag |= HAS_NULL; + } else { + ASSERT(0); + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case HAS_VALUE: { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT1(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { // has a value + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iv != iStart) { + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iv]], + iv < (pColData->nVal - 1) ? pColData->aOffset[iv + 1] - pColData->aOffset[iv] + : pColData->nData - pColData->aOffset[iv]); + } + // TODO + ASSERT(0); + } else { + if (iv != iStart) { + memcpy(&pColData->pData[TYPE_BYTES[pColData->type] * iStart], + &pColData->pData[TYPE_BYTES[pColData->type] * iv], TYPE_BYTES[pColData->type]); + } + memmove(&pColData->pData[TYPE_BYTES[pColData->type] * (iStart + 1)], + &pColData->pData[TYPE_BYTES[pColData->type] * iEnd], + TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + SET_BIT1(pColData->pBitMap, iStart, 1); + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + uint8_t flag = HAS_VALUE; + for (int32_t i = 0; i < pColData->nVal - (iEnd - iStart - 1); ++i) { + if (GET_BIT1(pColData->pBitMap, i) == 0) { + flag |= HAS_NONE; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } else { // all NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(&pColData->pData[pColData->aOffset[iStart]], &pColData->pData[pColData->aOffset[iEnd - 1]], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + case (HAS_VALUE | HAS_NULL): { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nDiff = pColData->aOffset[iEnd - 1] - pColData->aOffset[iStart]; + + memmove(pColData->pData + pColData->aOffset[iStart], pColData->pData + pColData->aOffset[iEnd - 1], + pColData->nData - pColData->aOffset[iEnd - 1]); + pColData->nData -= nDiff; + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + pColData->aOffset[j] = pColData->aOffset[i] - nDiff; + } + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * iStart, + pColData->pData + TYPE_BYTES[pColData->type] * (iEnd - 1), + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd + 1)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd - 1, j = iStart; i < pColData->nVal; ++i, ++j) { + SET_BIT1(pColData->pBitMap, j, GET_BIT1(pColData->pBitMap, i)); + } + + pColData->nVal -= (iEnd - iStart - 1); + + uint8_t flag = 0; + for (int32_t i = 0; i < pColData->nVal; ++i) { + if (GET_BIT1(pColData->pBitMap, i)) { + flag |= HAS_VALUE; + } else { + flag |= HAS_NULL; + } + + if (flag == pColData->flag) break; + } + pColData->flag = flag; + } break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): { + uint8_t bv; + int32_t iv; + for (int32_t i = iEnd - 1; i >= iStart; --i) { + bv = GET_BIT2(pColData->pBitMap, i); + if (bv) { + iv = i; + break; + } + } + + if (bv) { + // TODO + ASSERT(0); + } else { // ALL NONE + if (IS_VAR_DATA_TYPE(pColData->type)) { + // TODO + ASSERT(0); + } else { + memmove(pColData->pData + TYPE_BYTES[pColData->type] * (iStart + 1), + pColData->pData + TYPE_BYTES[pColData->type] * iEnd, + TYPE_BYTES[pColData->type] * (pColData->nVal - iEnd)); + pColData->nData -= (TYPE_BYTES[pColData->type] * (iEnd - iStart - 1)); + } + + for (int32_t i = iEnd, j = iStart + 1; i < pColData->nVal; ++i, ++j) { + SET_BIT2(pColData->pBitMap, j, GET_BIT2(pColData->pBitMap, i)); + } + } + pColData->nVal -= (iEnd - iStart - 1); + } break; + default: + ASSERT(0); + break; + } +} +static void tColDataMerge(SColData *aColData, int32_t nColData) { + int32_t iStart = 0; + for (;;) { + if (iStart >= aColData[0].nVal - 1) break; + + int32_t iEnd = iStart + 1; + while (iEnd < aColData[0].nVal) { + if (((TSKEY *)aColData[0].pData)[iEnd] != ((TSKEY *)aColData[0].pData)[iStart]) break; + + iEnd++; + } + + if (iEnd - iStart > 1) { + for (int32_t i = 0; i < nColData; i++) { + tColDataMergeImpl(&aColData[i], iStart, iEnd); + } + } + + iStart++; + } +} +void tColDataSortMerge(SArray *colDataArr) { + int32_t nColData = TARRAY_SIZE(colDataArr); + SColData *aColData = (SColData *)TARRAY_DATA(colDataArr); + + if (aColData[0].nVal <= 1) goto _exit; + + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].flag == HAS_VALUE); + + int8_t doSort = 0; + int8_t doMerge = 0; + // scan ------- + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] > aKey[iVal - 1]) { + continue; + } else if (aKey[iVal] < aKey[iVal - 1]) { + doSort = 1; + break; + } else { + doMerge = 1; + } + } + + // sort ------- + if (doSort) { + tColDataSort(aColData, nColData); + } + + if (doMerge != 1) { + for (int32_t iVal = 1; iVal < aColData[0].nVal; ++iVal) { + if (aKey[iVal] == aKey[iVal - 1]) { + doMerge = 1; + break; + } + } + } + + // merge ------- + if (doMerge) { + tColDataMerge(aColData, nColData); + } + +_exit: + return; +} + +int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT1_SIZE(pColData->nVal)); + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + if (pBuf) memcpy(pBuf + n, pColData->pBitMap, BIT2_SIZE(pColData->nVal)); + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; } - // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; + // value + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); + n += (pColData->nVal << 2); - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } else { + if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); + n += pColData->nData; + } + } - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + return n; +} + +int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(pBuf + n, &pColData->cid); + n += tGetI8(pBuf + n, &pColData->type); + n += tGetI32v(pBuf + n, &pColData->nVal); + n += tGetI8(pBuf + n, &pColData->flag); + + // bitmap + switch (pColData->flag) { + case (HAS_NULL | HAS_NONE): + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pColData->pBitMap = pBuf + n; + n += BIT1_SIZE(pColData->nVal); + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pColData->pBitMap = pBuf + n; + n += BIT2_SIZE(pColData->nVal); + break; + default: + break; } // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); + if (pColData->flag & HAS_VALUE) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + pColData->aOffset = (int32_t *)(pBuf + n); + n += (pColData->nVal << 2); -_exit: - return code; + n += tGetI32v(pBuf + n, &pColData->nData); + pColData->pData = pBuf + n; + n += pColData->nData; + } else { + pColData->pData = pBuf + n; + pColData->nData = TYPE_BYTES[pColData->type] * pColData->nVal; + n += pColData->nData; + } + } + + return n; } #define CALC_SUM_MAX_MIN(SUM, MAX, MIN, VAL) \ diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c8f1efa1ab4f6e398c3252f1fa1376747d1371cd..02d83af3a7e66396a0b1585ad0a823e618f558d4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -87,8 +87,8 @@ char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. // If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) -bool tsSmlDataFormat = false; -int32_t tsSmlBatchSize = 10000; +// bool tsSmlDataFormat = false; +// int32_t tsSmlBatchSize = 10000; // query int32_t tsQueryPolicy = 1; @@ -319,8 +319,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, true) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", "", 1) != 0) return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, 1) != 0) return -1; - if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; + // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, 1) != 0) return -1; + // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxMemUsedByInsert", tsMaxMemUsedByInsert, 1, INT32_MAX, true) != 0) return -1; if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; @@ -354,7 +354,9 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, 0) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; +#if !defined(_ALPINE) if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; +#endif if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; @@ -676,9 +678,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); - tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; - tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; + // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; tsMaxMemUsedByInsert = cfgGetItem(pCfg, "maxMemUsedByInsert")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; @@ -1069,10 +1071,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); } else if (strcasecmp("smlTagName", name) == 0) { tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); - } else if (strcasecmp("smlDataFormat", name) == 0) { - tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; - } else if (strcasecmp("smlBatchSize", name) == 0) { - tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; + // } else if (strcasecmp("smlDataFormat", name) == 0) { + // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; + // } else if (strcasecmp("smlBatchSize", name) == 0) { + // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; } else if (strcasecmp("shellActivityTimer", name) == 0) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; } else if (strcasecmp("supportVnodes", name) == 0) { @@ -1142,6 +1144,8 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; } else if (strcasecmp("uDebugFlag", name) == 0) { uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + } else if (strcasecmp("useAdapter", name) == 0) { + tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 44d3f7be25d5ee9eea280b1e089973ff459a11e7..e180959d1e1c73805265e144f7e38882b9ad9e7b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -918,6 +918,65 @@ int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) tDecoderClear(&decoder); return 0; } + +int32_t tSerializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stbName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->colName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->idxName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->idxType) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->colName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->idxName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->idxType) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} +int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + tEndEncode(&encoder); + + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + + return 0; +} int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3002,7 +3061,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp void tFreeSerializeSTableIndexRsp(STableIndexRsp *pRsp) { if (pRsp->pIndex != NULL) { - taosArrayDestroy(pRsp->pIndex); + tFreeSTableIndexRsp(pRsp); pRsp->pIndex = NULL; } } @@ -3192,6 +3251,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->filterTb) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; tEndEncode(&encoder); @@ -3208,6 +3268,7 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->filterTb) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; tEndDecode(&decoder); @@ -3982,14 +4043,15 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq tDecoderClear(&decoder); return 0; } - -int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { +int32_t tSerializeSDropIdxReq(void *buf, int32_t bufLen, SDropIndexReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->colName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1; + if (tEncodeI64(&encoder, pReq->stbUid) < 0) return -1; if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } @@ -3998,20 +4060,53 @@ int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq * int32_t tlen = encoder.pos; tEncoderClear(&encoder); return tlen; + // TODO + return 0; } - -int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { +int32_t tDeserializeSDropIdxReq(void *buf, int32_t bufLen, SDropIndexReq *pReq) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->colName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->stbUid) < 0) return -1; if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } tEndDecode(&decoder); + tDecoderClear(&decoder); + // TODO + return 0; +} + +int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI64(&encoder, pReq->compactStartTime) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->compactStartTime) < 0) return -1; + tEndDecode(&decoder); + tDecoderClear(&decoder); return 0; } @@ -4116,6 +4211,68 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode return 0; } +int32_t tSerializeSDisableVnodeWriteReq(void *buf, int32_t bufLen, SDisableVnodeWriteReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; + if (tEncodeI8(&encoder, pReq->disable) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDisableVnodeWriteReq(void *buf, int32_t bufLen, SDisableVnodeWriteReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->disable) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSAlterVnodeHashRangeReq(void *buf, int32_t bufLen, SAlterVnodeHashRangeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->srcVgId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dstVgId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->hashBegin) < 0) return -1; + if (tEncodeI32(&encoder, pReq->hashEnd) < 0) return -1; + if (tEncodeI64(&encoder, pReq->reserved) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAlterVnodeHashRangeReq(void *buf, int32_t bufLen, SAlterVnodeHashRangeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->srcVgId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dstVgId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->hashBegin) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->hashEnd) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->reserved) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -5425,6 +5582,16 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + + if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1; + if (tEncodeU64(&encoder, pReq->targetStbUid) < 0) return -1; + if (tEncodeI32(&encoder, taosArrayGetSize(pReq->fillNullCols)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pReq->fillNullCols); ++i) { + SColLocation *pCol = taosArrayGet(pReq->fillNullCols, i); + if (tEncodeI16(&encoder, pCol->slotId) < 0) return -1; + if (tEncodeI16(&encoder, pCol->colId) < 0) return -1; + if (tEncodeI8(&encoder, pCol->type) < 0) return -1; + } if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1; if (tEncodeI8(&encoder, pReq->igUpdate) < 0) return -1; @@ -5487,6 +5654,28 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea } } } + if (tDecodeI8(&decoder, &pReq->createStb) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->targetStbUid) < 0) return -1; + int32_t numOfFillNullCols = 0; + if (tDecodeI32(&decoder, &numOfFillNullCols) < 0) return -1; + if (numOfFillNullCols > 0) { + pReq->fillNullCols = taosArrayInit(numOfFillNullCols, sizeof(SColLocation)); + if (pReq->fillNullCols == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfFillNullCols; ++i) { + SColLocation col = {0}; + if (tDecodeI16(&decoder, &col.slotId) < 0) return -1; + if (tDecodeI16(&decoder, &col.colId) < 0) return -1; + if (tDecodeI8(&decoder, &col.type) < 0) return -1; + if (taosArrayPush(pReq->fillNullCols, &col) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + } if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igUpdate) < 0) return -1; @@ -5559,6 +5748,7 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosArrayDestroy(pReq->pTags); taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); + taosArrayDestroy(pReq->fillNullCols); } int32_t tEncodeSRSmaParam(SEncoder *pCoder, const SRSmaParam *pRSmaParam) { @@ -5633,30 +5823,6 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { return 0; } -STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) { - STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) { - return NULL; - } - - for (int i = 0; i < nCols; i++) { - SSchema *schema = pSchema + i; - if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - } - - STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); - if (pNSchema == NULL) { - tdDestroyTSchemaBuilder(&schemaBuilder); - return NULL; - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - return pNSchema; -} - int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (tStartEncode(pCoder) < 0) return -1; @@ -5734,6 +5900,25 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { return 0; } +void tDestroySVCreateTbReq(SVCreateTbReq *pReq, int32_t flags) { + if (pReq == NULL) return; + + if (flags & TSDB_MSG_FLG_ENCODE) { + // TODO + } else if (flags & TSDB_MSG_FLG_DECODE) { + if (pReq->comment) { + pReq->comment = NULL; + taosMemoryFree(pReq->comment); + } + + if (pReq->type == TSDB_CHILD_TABLE) { + if (pReq->ctb.tagName) taosArrayDestroy(pReq->ctb.tagName); + } else if (pReq->type == TSDB_NORMAL_TABLE) { + if (pReq->ntb.schemaRow.pSchema) taosMemoryFree(pReq->ntb.schemaRow.pSchema); + } + } +} + int tEncodeSVCreateTbBatchReq(SEncoder *pCoder, const SVCreateTbBatchReq *pReq) { int32_t nReq = taosArrayGetSize(pReq->pArray); @@ -6725,3 +6910,332 @@ int32_t tDecodeSBatchDeleteReq(SDecoder *pDecoder, SBatchDeleteReq *pReq) { } return 0; } + +static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pSubmitTbData->flags) < 0) return -1; + + // auto create table + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + ASSERT(pSubmitTbData->pCreateTbReq); + if (tEncodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) return -1; + } + + // submit data + if (tEncodeI64(pCoder, pSubmitTbData->suid) < 0) return -1; + if (tEncodeI64(pCoder, pSubmitTbData->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pSubmitTbData->sver) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); + + if (tEncodeU64v(pCoder, nColData) < 0) return -1; + + for (uint64_t i = 0; i < nColData; i++) { + pCoder->pos += tPutColData(pCoder->data ? pCoder->data + pCoder->pos : NULL, &aColData[i]); + } + } else { + if (tEncodeU64v(pCoder, TARRAY_SIZE(pSubmitTbData->aRowP)) < 0) return -1; + + SRow **rows = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + for (int32_t iRow = 0; iRow < TARRAY_SIZE(pSubmitTbData->aRowP); ++iRow) { + if (pCoder->data) memcpy(pCoder->data + pCoder->pos, rows[iRow], rows[iRow]->len); + pCoder->pos += rows[iRow]->len; + } + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { + int32_t code = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pSubmitTbData->flags) < 0) return -1; + + if (pSubmitTbData->flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + pSubmitTbData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (pSubmitTbData->pCreateTbReq == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + if (tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + // submit data + if (tDecodeI64(pCoder, &pSubmitTbData->suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI64(pCoder, &pSubmitTbData->uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + if (tDecodeI32v(pCoder, &pSubmitTbData->sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aCol = taosArrayInit(nColData, sizeof(SColData)); + if (pSubmitTbData->aCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nColData; ++i) { + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, taosArrayReserve(pSubmitTbData->aCol, 1)); + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); + if (pSubmitTbData->aRowP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow **ppRow = taosArrayReserve(pSubmitTbData->aRowP, 1); + + *ppRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += (*ppRow)->len; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + // TODO: clear + } + return 0; +} + +int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1; + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + if (tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) { + int32_t code = 0; + + memset(pReq, 0, sizeof(*pReq)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pReq->aSubmitTbData = taosArrayInit(nSubmitTbData, sizeof(SSubmitTbData)); + if (pReq->aSubmitTbData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (uint64_t i = 0; i < nSubmitTbData; i++) { + if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pReq->aSubmitTbData) { + // todo + taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; + } + } + return code; +} + +void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { + if (NULL == pTbData) { + return; + } + + if (flag == TSDB_MSG_FLG_ENCODE || flag == TSDB_MSG_FLG_CMPT) { + if (pTbData->pCreateTbReq) { + if (flag == TSDB_MSG_FLG_ENCODE) { + tdDestroySVCreateTbReq(pTbData->pCreateTbReq); + } else { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + } + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pTbData->aCol); + + for (int32_t i = 0; i < nColData; ++i) { + tColDataDestroy(&aColData[i]); + } + taosArrayDestroy(pTbData->aCol); + } else { + int32_t nRow = TARRAY_SIZE(pTbData->aRowP); + SRow **rows = (SRow **)TARRAY_DATA(pTbData->aRowP); + + for (int32_t i = 0; i < nRow; ++i) { + tRowDestroy(rows[i]); + } + taosArrayDestroy(pTbData->aRowP); + } + } else if (flag == TSDB_MSG_FLG_DECODE) { + if (pTbData->pCreateTbReq) { + tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + taosMemoryFree(pTbData->pCreateTbReq); + } + + if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + taosArrayDestroy(pTbData->aCol); + } else { + taosArrayDestroy(pTbData->aRowP); + } + } +} + +void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { + if (pReq->aSubmitTbData == NULL) return; + + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySSubmitTbData(&aSubmitTbData[i], flag); + } + taosArrayDestroy(pReq->aSubmitTbData); +} + +int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pRsp->affectedRows) < 0) return -1; + + if (tEncodeU64v(pCoder, taosArrayGetSize(pRsp->aCreateTbRsp)) < 0) return -1; + for (int32_t i = 0; i < taosArrayGetSize(pRsp->aCreateTbRsp); ++i) { + if (tEncodeSVCreateTbRsp(pCoder, taosArrayGet(pRsp->aCreateTbRsp, i)) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSSubmitRsp2(SDecoder *pCoder, SSubmitRsp2 *pRsp) { + int32_t code = 0; + + memset(pRsp, 0, sizeof(SSubmitRsp2)); + + // decode + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (tDecodeI32v(pCoder, &pRsp->affectedRows) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + uint64_t nCreateTbRsp; + if (tDecodeU64v(pCoder, &nCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + if (nCreateTbRsp) { + pRsp->aCreateTbRsp = taosArrayInit(nCreateTbRsp, sizeof(SVCreateTbRsp)); + if (pRsp->aCreateTbRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pRsp->aCreateTbRsp, 1); + if (tDecodeSVCreateTbRsp(pCoder, pCreateTbRsp) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + } + } + + tEndDecode(pCoder); + +_exit: + if (code) { + if (pRsp->aCreateTbRsp) { + taosArrayDestroyEx(pRsp->aCreateTbRsp, NULL /* todo */); + } + } + return code; +} + +void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { + if (NULL == pRsp) { + return; + } + + if (flag & TSDB_MSG_FLG_ENCODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } else if (flag & TSDB_MSG_FLG_DECODE) { + if (pRsp->aCreateTbRsp) { + int32_t nCreateTbRsp = TARRAY_SIZE(pRsp->aCreateTbRsp); + SVCreateTbRsp *aCreateTbRsp = TARRAY_DATA(pRsp->aCreateTbRsp); + for (int32_t i = 0; i < nCreateTbRsp; ++i) { + if (aCreateTbRsp[i].pMeta) { + taosMemoryFree(aCreateTbRsp[i].pMeta); + } + } + taosArrayDestroy(pRsp->aCreateTbRsp); + } + } +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f21938ed29632817d84d6908af119a7fc1c73ffe..e5ed7a3728cebe78ae9cc24304e6e8ce2e94c7ef 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -90,10 +90,8 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in SName* toName(int32_t acctId, const char* pDbName, const char* pTableName, SName* pName) { pName->type = TSDB_TABLE_NAME_T; pName->acctId = acctId; - memset(pName->dbname, 0, TSDB_DB_NAME_LEN); - strncpy(pName->dbname, pDbName, TSDB_DB_NAME_LEN - 1); - memset(pName->tname, 0, TSDB_TABLE_NAME_LEN); - strncpy(pName->tname, pTableName, TSDB_TABLE_NAME_LEN - 1); + snprintf(pName->dbname, sizeof(pName->dbname), "%s", pDbName); + snprintf(pName->tname, sizeof(pName->tname), "%s", pTableName); return pName; } @@ -162,9 +160,7 @@ int32_t tNameGetFullDbName(const SName* name, char* dst) { return 0; } -bool tNameIsEmpty(const SName* name) { - return name->type == 0 || name->acctId == 0; -} +bool tNameIsEmpty(const SName* name) { return name->type == 0 || name->acctId == 0; } const char* tNameGetTableName(const SName* name) { ASSERT(name != NULL && name->type == TSDB_TABLE_NAME_T); @@ -284,8 +280,8 @@ int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { } static int compareKv(const void* p1, const void* p2) { - SSmlKv* kv1 = *(SSmlKv**)p1; - SSmlKv* kv2 = *(SSmlKv**)p2; + SSmlKv* kv1 = (SSmlKv*)p1; + SSmlKv* kv2 = (SSmlKv*)p2; int32_t kvLen1 = kv1->keyLen; int32_t kvLen2 = kv2->keyLen; int32_t res = strncasecmp(kv1->key, kv2->key, TMIN(kvLen1, kvLen2)); @@ -302,11 +298,11 @@ static int compareKv(const void* p1, const void* p2) { void buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); - if(sb.buf == NULL) return; + if (sb.buf == NULL) return; taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); - SSmlKv* tagKv = taosArrayGetP(rName->tags, j); + SSmlKv* tagKv = taosArrayGet(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 969da4226fb3cf59e124432701d64a270f20c373..9d381ce15fa762b74cecf23eeec1fa6667a10b99 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -341,7 +341,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { } if (iColumn == 0) { - ASSERT(pColVal->cid == pTColumn->colId); + ASSERT(pColVal && pColVal->cid == pTColumn->colId); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID); } else { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index de225581a679db43a4c5b7aaccc12854d75d0495..b42c88ac9734f789c04e5eca49ea48959158a5a7 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -185,7 +185,7 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { pDst->arr = taosArrayInit(num, sizeof(char *)); for (size_t i = 0; i < num; i++) { char *p = (char *)taosArrayGetP(pSrc->arr, i); - char *n = strdup(p); + char *n = taosStrdup(p); taosArrayPush(pDst->arr, &n); } } else if (pSrc->nType == TSDB_DATA_TYPE_VALUE_ARRAY) { diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index b05ae602f8ee5299ed108756f73f648867d1f81a..4f8652d02cc0602583edff7f59354d9d093275fc 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -117,7 +117,7 @@ STSchema *genSTSchema(int16_t nCols) { } STSchema *pResult = NULL; - pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1); + pResult = tBuildTSchema(pSchema, nCols, 1); taosMemoryFree(pSchema); return pResult; diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 4910b0ac3faab414b4a3197cd2412a7e4badd21e..d3cffaa185382af4ddfa7db681a1b9357582fb96 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -29,6 +29,7 @@ #define DM_MACHINE_CODE "Get machine code." #define DM_VERSION "Print program version." #define DM_EMAIL "" +#define DM_MEM_DBG "Enable memory debug" // clang-format on static struct { #ifdef WINDOWS @@ -37,6 +38,7 @@ static struct { bool dumpConfig; bool dumpSdb; bool generateGrant; + bool memDbg; bool printAuth; bool printVersion; bool printHelp; @@ -166,8 +168,10 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } else if (strcmp(argv[i], "-e") == 0) { global.envCmd[cmdEnvIndex] = argv[++i]; cmdEnvIndex++; + } else if (strcmp(argv[i], "-dm") == 0) { + global.memDbg = true; } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || - strcmp(argv[i], "-?")) { + strcmp(argv[i], "-?") == 0) { global.printHelp = true; } else { } @@ -212,6 +216,7 @@ static void dmPrintHelp() { printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD); printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE); printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE); + printf("%s%s%s%s\n", indent, "-dm,", indent, DM_MEM_DBG); printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION); printf("\n\nReport bugs to %s.\n", DM_EMAIL); @@ -272,6 +277,18 @@ int mainWindows(int argc, char **argv) { return 0; } +#if defined(LINUX) + if (global.memDbg) { + int32_t code = taosMemoryDbgInit(); + if (code) { + printf("failed to init memory dbg, error:%s\n", tstrerror(code)); + return code; + } + tsAsyncLog = false; + printf("memory dbg enabled\n"); + } +#endif + if (dmInitLog() != 0) { printf("failed to start since init log error\n"); taosCleanupArgs(); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index ff32cbcb08b2c743644df3b077408dfd4878e999..600a1f6829e11a1d38ffa262e7d360687c67b90d 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -48,7 +48,7 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t dmProcessGrantReq(SRpcMsg *pMsg); +int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 76c8e09b70a31c8e9168809dea69bc1a1a3e5478..0d3b423771267cc567a4cfc95f2b550c508342d3 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -234,7 +234,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { code = dmProcessRetrieve(pMgmt, pMsg); break; case TDMT_MND_GRANT: - code = dmProcessGrantReq(pMsg); + code = dmProcessGrantReq(&pMgmt->pData->clusterId, pMsg); break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index cf4eaaf7d1b205d23360e93f444fd2b7495cc122..fe65c3dde96a0e81ab8211fa87aebc7e92ee0294 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -157,6 +157,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; @@ -182,6 +184,9 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 6e724f4d4322c3aee7a143b1cffaa4cae155d3e7..e3fa2964b74b697cd507a9fddcb27a848010e2e1 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -54,6 +54,7 @@ typedef struct { int32_t vgVersion; int32_t refCount; int8_t dropped; + int8_t disable; char *path; SVnode *pImpl; SMultiWorker pWriteW; @@ -80,13 +81,15 @@ typedef struct { SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId); void vmReleaseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl); -void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode); +void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal); // vmHandle.c SArray *vmGetMsgHandles(); int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); -int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); // vmFile.c int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 220e55f7f38862b0f787a1ee4e13faee0fae44c2..5cf408a905fd267ad982d93a409e4859eb691cfb 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -157,6 +157,7 @@ static int32_t vmTsmaAdjustDays(SVnodeCfg *pCfg, SCreateVnodeReq *pReq) { return 0; } +#if 0 static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) { if (pReq->isTsma) { SMsgHead *smaMsg = pReq->pTsma; @@ -165,6 +166,7 @@ static int32_t vmTsmaProcessCreate(SVnode *pVnode, SCreateVnodeReq *pReq) { } return 0; } +#endif int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SCreateVnodeReq req = {0}; @@ -245,12 +247,14 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { goto _OVER; } +#if 0 code = vmTsmaProcessCreate(pImpl, &req); if (code != 0) { dError("vgId:%d, failed to create tsma since %s", req.vgId, terrstr()); code = terrno; goto _OVER; } +#endif code = vnodeStart(pImpl); if (code != 0) { @@ -277,7 +281,94 @@ _OVER: return code; } -int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { +int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + SDisableVnodeWriteReq req = {0}; + if (tDeserializeSDisableVnodeWriteReq(pMsg->pCont, pMsg->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("vgId:%d, vnode write disable:%d", req.vgId, req.disable); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to disable write since %s", req.vgId, terrstr()); + terrno = TSDB_CODE_VND_NOT_EXIST; + return -1; + } + + pVnode->disable = req.disable; + vmReleaseVnode(pMgmt, pVnode); + return 0; +} + +int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + SAlterVnodeHashRangeReq req = {0}; + if (tDeserializeSAlterVnodeHashRangeReq(pMsg->pCont, pMsg->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t srcVgId = req.srcVgId; + int32_t dstVgId = req.dstVgId; + dInfo("vgId:%d, start to alter vnode hashrange:[%u, %u], dstVgId:%d", req.srcVgId, req.hashBegin, req.hashEnd, + req.dstVgId); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, srcVgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to alter hashrange since %s", srcVgId, terrstr()); + terrno = TSDB_CODE_VND_NOT_EXIST; + return -1; + } + + SWrapperCfg wrapperCfg = { + .dropped = pVnode->dropped, + .vgId = dstVgId, + .vgVersion = pVnode->vgVersion, + }; + tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); + + dInfo("vgId:%d, close vnode", srcVgId); + vmCloseVnode(pMgmt, pVnode, true); + + char srcPath[TSDB_FILENAME_LEN] = {0}; + char dstPath[TSDB_FILENAME_LEN] = {0}; + snprintf(srcPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, srcVgId); + snprintf(dstPath, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, dstVgId); + + dInfo("vgId:%d, alter vnode hashrange at %s", srcVgId, srcPath); + if (vnodeAlterHashRange(srcPath, dstPath, &req, pMgmt->pTfs) < 0) { + dError("vgId:%d, failed to alter vnode hashrange since %s", srcVgId, terrstr()); + return -1; + } + + dInfo("vgId:%d, open vnode", dstVgId); + SVnode *pImpl = vnodeOpen(dstPath, pMgmt->pTfs, pMgmt->msgCb); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode at %s since %s", dstVgId, dstPath, terrstr()); + return -1; + } + + if (vmOpenVnode(pMgmt, &wrapperCfg, pImpl) != 0) { + dError("vgId:%d, failed to open vnode mgmt since %s", dstVgId, terrstr()); + return -1; + } + + if (vnodeStart(pImpl) != 0) { + dError("vgId:%d, failed to start sync since %s", dstVgId, terrstr()); + return -1; + } + + if (vmWriteVnodeListToFile(pMgmt) != 0) { + dError("vgId:%d, failed to write vnode list since %s", dstVgId, terrstr()); + return -1; + } + + dInfo("vgId:%d, vnode hashrange is altered", dstVgId); + return 0; +} + +int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SAlterVnodeReplicaReq alterReq = {0}; if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -285,16 +376,16 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vgId = alterReq.vgId; - dInfo("vgId:%d, start to alter vnode, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, - alterReq.selfIndex, alterReq.strict); + dInfo("vgId:%d, start to alter vnode replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex, + alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { SReplica *pReplica = &alterReq.replicas[i]; - dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", alterReq.vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { terrno = TSDB_CODE_INVALID_MSG; - dError("vgId:%d, failed to alter replica since invalid msg", alterReq.vgId); + dError("vgId:%d, failed to alter replica since invalid msg", vgId); return -1; } @@ -302,7 +393,7 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; - dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", alterReq.vgId, pReplica->id, pReplica->fqdn, + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", vgId, pReplica->id, pReplica->fqdn, pReplica->port); return -1; } @@ -321,18 +412,18 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { .vgVersion = pVnode->vgVersion, }; tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); - vmCloseVnode(pMgmt, pVnode); + vmCloseVnode(pMgmt, pVnode, false); char path[TSDB_FILENAME_LEN] = {0}; snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); - if (vnodeAlter(path, &alterReq, pMgmt->pTfs) < 0) { + if (vnodeAlterReplica(path, &alterReq, pMgmt->pTfs) < 0) { dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); return -1; } - dInfo("vgId:%d, start to open vnode", vgId); + dInfo("vgId:%d, close vnode", vgId); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); @@ -383,7 +474,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - vmCloseVnode(pMgmt, pVnode); + vmCloseVnode(pMgmt, pVnode, false); vmWriteVnodeListToFile(pMgmt); dInfo("vgId:%d, is dropped", vgId); @@ -431,6 +522,8 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -447,7 +540,8 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 951544c76625ab8091cadd6e944cd702ca5e3a44..8008e5f81031af617a169f1b4414338f71de897e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -53,7 +53,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { pVnode->vgVersion = pCfg->vgVersion; pVnode->refCount = 0; pVnode->dropped = 0; - pVnode->path = tstrdup(pCfg->path); + pVnode->path = taosStrdup(pCfg->path); pVnode->pImpl = pImpl; if (pVnode->path == NULL) { @@ -76,7 +76,7 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return code; } -void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { +void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) { char path[TSDB_FILENAME_LEN] = {0}; taosThreadRwlockWrlock(&pMgmt->lock); @@ -122,10 +122,26 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { vnodePostClose(pVnode->pImpl); vmFreeQueue(pMgmt, pVnode); + + if (commitAndRemoveWal) { + dInfo("vgId:%d, commit data", pVnode->vgId); + vnodeSyncCommit(pVnode->pImpl); + vnodeBegin(pVnode->pImpl); + dInfo("vgId:%d, commit data finished", pVnode->vgId); + } + vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; dInfo("vgId:%d, vnode is closed", pVnode->vgId); + if (commitAndRemoveWal) { + char path[TSDB_FILENAME_LEN] = {0}; + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d%swal", TD_DIRSEP, pVnode->vgId, TD_DIRSEP); + dInfo("vgId:%d, remove all wals, path:%s", pVnode->vgId, path); + tfsRmdir(pMgmt->pTfs, path); + tfsMkdir(pMgmt->pTfs, path); + } + if (pVnode->dropped) { dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId); @@ -255,7 +271,7 @@ static void *vmCloseVnodeInThread(void *param) { pMgmt->state.openVnodes, pMgmt->state.totalVnodes); tmsgReportStartup("vnode-close", stepDesc); - vmCloseVnode(pMgmt, pVnode); + vmCloseVnode(pMgmt, pVnode, false); } dInfo("thread:%d, numOfVnodes:%d is closed", pThread->threadIndex, pThread->vnodeNum); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 8f091c82d983137974cc7f6766fd85a8fca4d43f..c1b3cde9ea9342ced75f4747881fefae86104af1 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -41,7 +41,13 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { code = vmProcessDropVnodeReq(pMgmt, pMsg); break; case TDMT_VND_ALTER_REPLICA: - code = vmProcessAlterVnodeReq(pMgmt, pMsg); + code = vmProcessAlterVnodeReplicaReq(pMgmt, pMsg); + break; + case TDMT_VND_DISABLE_WRITE: + code = vmProcessDisableVnodeWriteReq(pMgmt, pMsg); + break; + case TDMT_VND_ALTER_HASHRANGE: + code = vmProcessAlterHashRangeReq(pMgmt, pMsg); break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; @@ -51,7 +57,7 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { if (IsReq(pMsg)) { if (code != 0) { if (terrno != 0) code = terrno; - dGError("msg:%p, failed to process since %s", pMsg, terrstr(code)); + dGError("msg:%p, failed to process since %s, type:%s", pMsg, terrstr(code), TMSG_INFO(pMsg->msgType)); } vmSendRsp(pMsg, code); } @@ -190,14 +196,21 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp terrno = TSDB_CODE_NO_DISKSPACE; code = terrno; dError("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); - } else if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + break; + } + if (pMsg->msgType == TDMT_VND_SUBMIT && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { terrno = TSDB_CODE_VND_NO_WRITE_AUTH; code = terrno; dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr(code)); - } else { - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteW.queue, pMsg); + break; + } + if (pMsg->msgType != TDMT_VND_ALTER_CONFIRM && pVnode->disable) { + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since its disable", pVnode->vgId, pMsg); + terrno = TSDB_CODE_VND_STOPPED; + break; } + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteW.queue, pMsg); break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index b6cce249ea935a55e0515a84955702e5e463ca4d..d8841201478b948f396e8d62a64c714290559c96 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -79,6 +79,13 @@ static void dmClearVars(SDnode *pDnode) { SDnodeData *pData = &pDnode->data; taosThreadRwlockWrlock(&pData->lock); + if (pData->oldDnodeEps != NULL) { + if (dmWriteEps(pData) == 0) { + dmRemoveDnodePairs(pData); + } + taosArrayDestroy(pData->oldDnodeEps); + pData->oldDnodeEps = NULL; + } if (pData->dnodeEps != NULL) { taosArrayDestroy(pData->dnodeEps); pData->dnodeEps = NULL; @@ -117,7 +124,7 @@ int32_t dmInitDnode(SDnode *pDnode) { taosThreadRwlockInit(&pWrapper->lock, NULL); snprintf(path, sizeof(path), "%s%s%s", tsDataDir, TD_DIRSEP, pWrapper->name); - pWrapper->path = strdup(path); + pWrapper->path = taosStrdup(path); if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 08330e025f2f496fdbe904450b2a9a659d6d0198..16931ab6dfa1a67f37792b19946c337461108062 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -109,9 +109,8 @@ static int32_t dmStartNodes(SDnode *pDnode) { } } - dInfo("TDengine initialized successfully"); - dmReportStartup("TDengine", "initialized successfully"); - + dInfo("The daemon initialized successfully"); + dmReportStartup("The daemon", "initialized successfully"); return 0; } @@ -143,7 +142,7 @@ int32_t dmRunDnode(SDnode *pDnode) { while (1) { if (pDnode->stop) { - dInfo("TDengine is about to stop"); + dInfo("The daemon is about to stop"); dmSetStatus(pDnode, DND_STAT_STOPPED); dmStopNodes(pDnode); dmCloseNodes(pDnode); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 3a1ca161a994ebd65a0471a13f49837eaeab48d4..23a047d49aff5c76c5b04140c7301898e6052478 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -39,7 +39,7 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; if (msgFp == NULL) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dGError("msg:%p, not processed since no handler", pMsg); + dGError("msg:%p, not processed since no handler, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); return -1; } @@ -93,18 +93,30 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { break; } - if (pDnode->status != DND_STAT_RUNNING) { - if (pRpc->msgType == TDMT_DND_SERVER_STATUS) { - dmProcessServerStartupStatus(pDnode, pRpc); - return; - } else { - if (pDnode->status == DND_STAT_INIT) { - terrno = TSDB_CODE_APP_IS_STARTING; +/* +pDnode is null, TD-22618 +at trans.c line 91 +before this line, dmProcessRpcMsg callback is set +after this line, parent is set +so when dmProcessRpcMsg is called, pDonde is still null. +*/ + if (pDnode != NULL){ + if(pDnode->status != DND_STAT_RUNNING) { + if (pRpc->msgType == TDMT_DND_SERVER_STATUS) { + dmProcessServerStartupStatus(pDnode, pRpc); + return; } else { - terrno = TSDB_CODE_APP_IS_STOPPING; + if (pDnode->status == DND_STAT_INIT) { + terrno = TSDB_CODE_APP_IS_STARTING; + } else { + terrno = TSDB_CODE_APP_IS_STOPPING; + } + goto _OVER; } - goto _OVER; - } + } + } else { + terrno = TSDB_CODE_APP_IS_STARTING; + goto _OVER; } if (pRpc->pCont == NULL && (IsReq(pRpc) || pRpc->contLen != 0)) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 784bb3c5e132cc5d012521ced390742009ad3262..c2f403dfbb4e827cb137a26056e8d7cec41fcf0a 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -168,7 +168,8 @@ void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps); void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet); void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); -void dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); +bool dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port); +void dmRemoveDnodePairs(SDnodeData *pData); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/node_util/src/dmEps.c b/source/dnode/mgmt/node_util/src/dmEps.c index 4285eb5c0786f75c25850e5c18c2689ab17869b5..e9ab8a04609b37593bd5007382b89263ac2de2bb 100644 --- a/source/dnode/mgmt/node_util/src/dmEps.c +++ b/source/dnode/mgmt/node_util/src/dmEps.c @@ -18,9 +18,18 @@ #include "tjson.h" #include "tmisce.h" -static void dmPrintEps(SDnodeData *pData); -static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep); -static void dmResetEps(SDnodeData *pData, SArray *dnodeEps); +typedef struct { + int32_t id; + uint16_t oldPort; + uint16_t newPort; + char oldFqdn[TSDB_FQDN_LEN]; + char newFqdn[TSDB_FQDN_LEN]; +} SDnodeEpPair; + +static void dmPrintEps(SDnodeData *pData); +static bool dmIsEpChanged(SDnodeData *pData, int32_t dnodeId, const char *ep); +static void dmResetEps(SDnodeData *pData, SArray *dnodeEps); +static int32_t dmReadDnodePairs(SDnodeData *pData); static void dmGetDnodeEp(SDnodeData *pData, int32_t dnodeId, char *pEp, char *pFqdn, uint16_t *pPort) { taosThreadRwlockRdlock(&pData->lock); @@ -137,7 +146,7 @@ int32_t dmReadEps(SDnodeData *pData) { } code = 0; - dInfo("succceed to read mnode file %s", file); + dInfo("succceed to read dnode file %s", file); _OVER: if (content != NULL) taosMemoryFree(content); @@ -146,6 +155,7 @@ _OVER: if (code != 0) { dError("failed to read dnode file:%s since %s", file, terrstr()); + return code; } if (taosArrayGetSize(pData->dnodeEps) == 0) { @@ -155,11 +165,16 @@ _OVER: taosArrayPush(pData->dnodeEps, &dnodeEp); } + if (dmReadDnodePairs(pData) != 0) { + return -1; + } + dDebug("reset dnode list on startup"); dmResetEps(pData, pData->dnodeEps); - if (dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) { + if (pData->oldDnodeEps == NULL && dmIsEpChanged(pData, pData->dnodeId, tsLocalEp)) { dError("localEp %s different with %s and need reconfigured", tsLocalEp, file); + terrno = TSDB_CODE_INVALID_CFG; return -1; } @@ -222,7 +237,8 @@ int32_t dmWriteEps(SDnodeData *pData) { code = 0; pData->updateTime = taosGetTimestampMs(); - dInfo("succeed to write dnode file:%s, dnodeVer:%" PRId64, realfile, pData->dnodeVer); + dInfo("succeed to write dnode file:%s, num:%d ver:%" PRId64, realfile, (int32_t)taosArrayGetSize(pData->dnodeEps), + pData->dnodeVer); _OVER: if (pJson != NULL) tjsonDelete(pJson); @@ -332,7 +348,8 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) { } } -void dmUpdateDnodeInfo(void *data, int32_t *did, int64_t *clusterId, char *fqdn, uint16_t *port) { +bool dmUpdateDnodeInfo(void *data, int32_t *did, int64_t *clusterId, char *fqdn, uint16_t *port) { + bool updated = false; SDnodeData *pData = data; int32_t dnodeId = -1; if (did != NULL) dnodeId = *did; @@ -342,11 +359,12 @@ void dmUpdateDnodeInfo(void *data, int32_t *did, int64_t *clusterId, char *fqdn, if (pData->oldDnodeEps != NULL) { int32_t size = (int32_t)taosArrayGetSize(pData->oldDnodeEps); for (int32_t i = 0; i < size; ++i) { - SDnodeEp *pDnodeEp = taosArrayGet(pData->oldDnodeEps, i); - if (strcmp(pDnodeEp->ep.fqdn, fqdn) == 0 && pDnodeEp->ep.port == *port) { - dInfo("dnode:%d, update ep:%s:%u to %s:%u", dnodeId, fqdn, *port, pDnodeEp->ep.fqdn, pDnodeEp->ep.port); - tstrncpy(fqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); - *port = pDnodeEp->ep.port; + SDnodeEpPair *pair = taosArrayGet(pData->oldDnodeEps, i); + if (strcmp(pair->oldFqdn, fqdn) == 0 && pair->oldPort == *port) { + dInfo("dnode:%d, update ep:%s:%u to %s:%u", dnodeId, fqdn, *port, pair->newFqdn, pair->newPort); + tstrncpy(fqdn, pair->newFqdn, TSDB_FQDN_LEN); + *port = pair->newPort; + updated = true; } } } @@ -370,10 +388,170 @@ void dmUpdateDnodeInfo(void *data, int32_t *did, int64_t *clusterId, char *fqdn, dInfo("dnode:%d, update ep:%s:%u to %s:%u", dnodeId, fqdn, *port, pDnodeEp->ep.fqdn, pDnodeEp->ep.port); tstrncpy(fqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); *port = pDnodeEp->ep.port; + updated = true; } if (clusterId != NULL) *clusterId = pData->clusterId; } } taosThreadRwlockUnlock(&pData->lock); -} \ No newline at end of file + return updated; +} + +static int32_t dmDecodeEpPairs(SJson *pJson, SDnodeData *pData) { + int32_t code = 0; + + SJson *dnodes = tjsonGetObjectItem(pJson, "dnodes"); + if (dnodes == NULL) return -1; + int32_t numOfDnodes = tjsonGetArraySize(dnodes); + + for (int32_t i = 0; i < numOfDnodes; ++i) { + SJson *dnode = tjsonGetArrayItem(dnodes, i); + if (dnode == NULL) return -1; + + SDnodeEpPair pair = {0}; + tjsonGetInt32ValueFromDouble(dnode, "id", pair.id, code); + if (code < 0) return -1; + code = tjsonGetStringValue(dnode, "fqdn", pair.oldFqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(dnode, "port", pair.oldPort, code); + if (code < 0) return -1; + code = tjsonGetStringValue(dnode, "new_fqdn", pair.newFqdn); + if (code < 0) return -1; + tjsonGetUInt16ValueFromDouble(dnode, "new_port", pair.newPort, code); + if (code < 0) return -1; + + if (taosArrayPush(pData->oldDnodeEps, &pair) == NULL) return -1; + } + + return code; +} + +void dmRemoveDnodePairs(SDnodeData *pData) { + char file[PATH_MAX] = {0}; + char bak[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); + snprintf(bak, sizeof(bak), "%s%sdnode%sep.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP); + dInfo("dnode file:%s is rename to bak file", file); + (void)taosRenameFile(file, bak); +} + +static int32_t dmReadDnodePairs(SDnodeData *pData) { + int32_t code = -1; + TdFilePtr pFile = NULL; + char *content = NULL; + SJson *pJson = NULL; + char file[PATH_MAX] = {0}; + snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); + + if (taosStatFile(file, NULL, NULL) < 0) { + dDebug("dnode file:%s not exist", file); + code = 0; + goto _OVER; + } + + pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to open dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + int64_t size = 0; + if (taosFStatFile(pFile, &size, NULL) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to fstat dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + content = taosMemoryMalloc(size + 1); + if (content == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + if (taosReadFile(pFile, content, size) != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to read dnode file:%s since %s", file, terrstr()); + goto _OVER; + } + + content[size] = '\0'; + + pJson = tjsonParse(content); + if (pJson == NULL) { + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + pData->oldDnodeEps = taosArrayInit(1, sizeof(SDnodeEpPair)); + if (pData->oldDnodeEps == NULL) { + dError("failed to calloc dnodeEp array since %s", strerror(errno)); + goto _OVER; + } + + if (dmDecodeEpPairs(pJson, pData) < 0) { + taosArrayDestroy(pData->oldDnodeEps); + pData->oldDnodeEps = NULL; + terrno = TSDB_CODE_INVALID_JSON_FORMAT; + goto _OVER; + } + + code = 0; + dInfo("succceed to read dnode file %s", file); + +_OVER: + if (content != NULL) taosMemoryFree(content); + if (pJson != NULL) cJSON_Delete(pJson); + if (pFile != NULL) taosCloseFile(&pFile); + + if (code != 0) { + dError("failed to read dnode file:%s since %s", file, terrstr()); + return code; + } + + // update old fqdn and port + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->oldDnodeEps); ++i) { + SDnodeEpPair *pair = taosArrayGet(pData->oldDnodeEps, i); + for (int32_t j = 0; j < (int32_t)taosArrayGetSize(pData->dnodeEps); ++j) { + SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, j); + if (pDnodeEp->id == pair->id) { + tstrncpy(pair->oldFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); + pair->oldPort = pDnodeEp->ep.port; + } + } + } + + // check new fqdn and port + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->oldDnodeEps); ++i) { + SDnodeEpPair *pair = taosArrayGet(pData->oldDnodeEps, i); + for (int32_t j = 0; j < (int32_t)taosArrayGetSize(pData->dnodeEps); ++j) { + SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, j); + if (pDnodeEp->id != pair->id && + (strcmp(pDnodeEp->ep.fqdn, pair->newFqdn) == 0 && pDnodeEp->ep.port == pair->newPort)) { + dError("dnode:%d, can't update ep:%s:%u to %s:%u since already exists as dnode:%d", pair->id, pair->oldFqdn, + pair->oldPort, pair->newFqdn, pair->newPort, pDnodeEp->id); + taosArrayDestroy(pData->oldDnodeEps); + pData->oldDnodeEps = NULL; + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + } + } + + for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->oldDnodeEps); ++i) { + SDnodeEpPair *pair = taosArrayGet(pData->oldDnodeEps, i); + for (int32_t j = 0; j < (int32_t)taosArrayGetSize(pData->dnodeEps); ++j) { + SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, j); + if (strcmp(pDnodeEp->ep.fqdn, pair->oldFqdn) == 0 && pDnodeEp->ep.port == pair->oldPort) { + dInfo("dnode:%d, will update ep:%s:%u to %s:%u", pDnodeEp->id, pDnodeEp->ep.fqdn, pDnodeEp->ep.port, + pair->newFqdn, pair->newPort); + tstrncpy(pDnodeEp->ep.fqdn, pair->newFqdn, TSDB_FQDN_LEN); + pDnodeEp->ep.port = pair->newPort; + } + } + } + + pData->dnodeVer = 0; + return 0; +} diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index cea0a43b61bf53cd1bcc47cae52000295e80fd90..9edfd9bf3b2e35c30639ad69ee84e3b1446271e3 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -30,6 +30,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); +SSdbRaw *mndDbActionEncode(SDbObj *pDb); const char *mndGetDbStr(const char *src); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6f6f801c39a6e8d292d3adf4b52840b0f747ed66..ac0c68f652635f49cd8396944fbe1db98e370bb9 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -328,6 +328,7 @@ typedef struct { SDbCfg cfg; SRWLatch lock; int64_t stateTs; + int64_t compactStartTime; } SDbObj; typedef struct { @@ -388,6 +389,18 @@ typedef struct { SSchemaWrapper schemaTag; // for dstVgroup } SSmaObj; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char dstTbName[TSDB_TABLE_FNAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; +} SIdxObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; @@ -444,6 +457,7 @@ typedef struct { STableMetaRsp* pMeta; bool sysDbRsp; char db[TSDB_DB_FNAME_LEN]; + char filterTb[TSDB_TABLE_NAME_LEN]; } SShowObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h new file mode 100644 index 0000000000000000000000000000000000000000..2d5479bc9b84bad7b65d9e6574a247193c854537 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndIndex.h @@ -0,0 +1,34 @@ +#ifndef _TD_MND_IDX_H_ +#define _TD_MND_IDX_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitIdx(SMnode *pMnode); +void mndCleanupIdx(SMnode *pMnode); +SIdxObj *mndAcquireIdx(SMnode *pMnode, char *Name); +void mndReleaseIdx(SMnode *pMnode, SIdxObj *pSma); +int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *pIdx); +int32_t mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); + +int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq); + +int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); + +int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_IDX_H_*/ \ No newline at end of file diff --git a/include/libs/cache/cache.h b/source/dnode/mnode/impl/inc/mndIndexComm.h similarity index 65% rename from include/libs/cache/cache.h rename to source/dnode/mnode/impl/inc/mndIndexComm.h index 6a2587ee9674668861d050ee734987125674e643..68112f0a2dce75bca0f2bd4295dbe1ea1b851115 100644 --- a/include/libs/cache/cache.h +++ b/source/dnode/mnode/impl/inc/mndIndexComm.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,15 +14,30 @@ * along with this program. If not, see . */ -#ifndef _TD_CACHE_H_ -#define _TD_CACHE_H_ +#ifndef _TD_MND_IDX_COMM_H_ +#define _TD_MND_IDX_COMM_H_ + +#include "mndInt.h" #ifdef __cplusplus extern "C" { #endif +typedef struct SSIdx { + int type; // sma or idx + void *pIdx; +} SSIdx; + +// retrieve sma index and tag index +typedef struct { + void *pSmaIter; + void *pIdxIter; +} SSmaAndTagIter; + +int32_t mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx); + #ifdef __cplusplus } #endif -#endif /*_TD_CACHE_H_*/ \ No newline at end of file +#endif /*_TD_MND_IDX_COMM_H_*/ \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 6a4031692301cfe835a2e6426eb51ca874d642ee..ac0924aab9965de8ccf25d135e5f4b2082121cbe 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -42,6 +42,11 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t const char *mndGetStbStr(const char *src); +int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId); +void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void *alterOriData, + int32_t alterOriDataLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 765d4fd33da9fa52266a89e8fc57d247c6494a96..51eb24f40235e21a811ef48a172c75ba09b4f396 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -43,6 +43,7 @@ int32_t mndAddDropVnodeAction(SMnode *, STrans *pTrans, SDbObj *pDb, SVgObj *pVg int32_t mndSetMoveVgroupsInfoToTrans(SMnode *, STrans *pTrans, int32_t dropDnodeId, bool force); int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pVgroup, SArray *pArray); +int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs); void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index ff131b50a6a8dff115426bfa47f24fdd97467fca..6621b6a3f9e0efe1536172607441c11e34eeacd6 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -100,8 +100,8 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { return 0; } - mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, - pConsumer->status, mndConsumerStatusName(pConsumer->status)); + mInfo("process consumer lost msg, consumer:0x%" PRIx64 " status:%d(%s)", pLostMsg->consumerId, pConsumer->status, + mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__READY) { mndReleaseConsumer(pMnode, pConsumer); @@ -141,7 +141,10 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerRecoverMsg *pRecoverMsg = pMsg->pCont; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + mError("cannot find consumer %" PRId64 " when processing consumer recover msg", pRecoverMsg->consumerId); + return -1; + } mInfo("receive consumer recover msg, consumer:0x%" PRIx64 " status:%d(%s)", pRecoverMsg->consumerId, pConsumer->status, mndConsumerStatusName(pConsumer->status)); @@ -256,9 +259,9 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - mDebug("check for consumer:0x%"PRIx64" status:%d(%s), sub-time:%"PRId64", uptime:%"PRId64", hbstatus:%d", - pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->upTime, - hbStatus); + mDebug("check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", uptime:%" PRId64 ", hbstatus:%d", + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->upTime, + hbStatus); if (status == MQ_CONSUMER_STATUS__READY) { if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { @@ -354,7 +357,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int64_t consumerId = req.consumerId; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); if (pConsumer == NULL) { - mError("consumer:0x%"PRIx64 " not exist", consumerId); + mError("consumer:0x%" PRIx64 " not exist", consumerId); terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } @@ -364,7 +367,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { - mInfo("try to recover consumer:0x%"PRIx64 "", consumerId); + mInfo("try to recover consumer:0x%" PRIx64 "", consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -416,7 +419,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { - mInfo("try to recover consumer:0x%"PRIx64, consumerId); + mInfo("try to recover consumer:0x%" PRIx64, consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); pRecoverMsg->consumerId = consumerId; @@ -431,7 +434,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { #endif if (status != MQ_CONSUMER_STATUS__READY) { - mInfo("consumer:0x%"PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); + mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; } @@ -441,7 +444,8 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2. check epoch, only send ep info when epochs do not match if (epoch != serverEpoch) { taosRLockLatch(&pConsumer->lock); - mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", consumerId, epoch, serverEpoch); + mInfo("process ask ep, consumer:0x%" PRIx64 "(epoch %d) update with server epoch %d", consumerId, epoch, + serverEpoch); int32_t numOfTopics = taosArrayGetSize(pConsumer->currentTopics); rsp.topics = taosArrayInit(numOfTopics, sizeof(SMqSubTopicEp)); @@ -455,9 +459,8 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { for (int32_t i = 0; i < numOfTopics; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); - // txn guarantees pSub is created - ASSERT(pSub); + taosRLockLatch(&pSub->lock); SMqSubTopicEp topicEp = {0}; @@ -465,7 +468,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2.1 fetch topic schema SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - ASSERT(pTopic); taosRLockLatch(&pTopic->lock); tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); topicEp.schema.nCols = pTopic->schema.nCols; @@ -587,7 +589,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { uint64_t consumerId = subscribe.consumerId; char *cgroup = subscribe.cgroup; - SMqConsumerObj *pConsumerOld = NULL; + SMqConsumerObj *pExistedConsumer = NULL; SMqConsumerObj *pConsumerNew = NULL; int32_t code = -1; @@ -608,8 +610,8 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto _over; } - pConsumerOld = mndAcquireConsumer(pMnode, consumerId); - if (pConsumerOld == NULL) { + pExistedConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pExistedConsumer == NULL) { mInfo("receive subscribe request from new consumer:0x%" PRIx64" cgroup:%s", consumerId, subscribe.cgroup); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); @@ -620,7 +622,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { subscribe.topicNames = NULL; for (int32_t i = 0; i < newTopicNum; i++) { - char *newTopicCopy = strdup(taosArrayGetP(pTopicList, i)); + char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, i)); taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy); } @@ -628,12 +630,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) goto _over; } else { - /*taosRLockLatch(&pConsumerOld->lock);*/ + /*taosRLockLatch(&pExistedConsumer->lock);*/ + int32_t status = atomic_load_32(&pExistedConsumer->status); - int32_t status = atomic_load_32(&pConsumerOld->status); - - mInfo("receive subscribe request from existing consumer:0x%" PRIx64 ", current status: %s, subscribe topic num: %d", - consumerId, mndConsumerStatusName(status), newTopicNum); + mInfo("receive subscribe request from existed consumer:0x%" PRIx64 " cgroup:%s, current status:%d(%s), subscribe topic num: %d", + consumerId, subscribe.cgroup, status,mndConsumerStatusName(status), newTopicNum); if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; @@ -642,35 +643,35 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); if (pConsumerNew == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; goto _over; } + pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; for (int32_t i = 0; i < newTopicNum; i++) { - char *newTopicCopy = strdup(taosArrayGetP(pTopicList, i)); + char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, i)); taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy); } int32_t oldTopicNum = 0; - if (pConsumerOld->currentTopics) { - oldTopicNum = taosArrayGetSize(pConsumerOld->currentTopics); + if (pExistedConsumer->currentTopics) { + oldTopicNum = taosArrayGetSize(pExistedConsumer->currentTopics); } int32_t i = 0, j = 0; while (i < oldTopicNum || j < newTopicNum) { if (i >= oldTopicNum) { - char *newTopicCopy = strdup(taosArrayGetP(pTopicList, j)); + char *newTopicCopy = taosStrdup(taosArrayGetP(pTopicList, j)); taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); j++; continue; } else if (j >= newTopicNum) { - char *oldTopicCopy = strdup(taosArrayGetP(pConsumerOld->currentTopics, i)); + char *oldTopicCopy = taosStrdup(taosArrayGetP(pExistedConsumer->currentTopics, i)); taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); i++; continue; } else { - char *oldTopic = taosArrayGetP(pConsumerOld->currentTopics, i); + char *oldTopic = taosArrayGetP(pExistedConsumer->currentTopics, i); char *newTopic = taosArrayGetP(pTopicList, j); int comp = strcmp(oldTopic, newTopic); if (comp == 0) { @@ -678,12 +679,12 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { j++; continue; } else if (comp < 0) { - char *oldTopicCopy = strdup(oldTopic); + char *oldTopicCopy = taosStrdup(oldTopic); taosArrayPush(pConsumerNew->rebRemovedTopics, &oldTopicCopy); i++; continue; } else { - char *newTopicCopy = strdup(newTopic); + char *newTopicCopy = taosStrdup(newTopic); taosArrayPush(pConsumerNew->rebNewTopics, &newTopicCopy); j++; continue; @@ -691,7 +692,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } } - if (pConsumerOld && taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 && + if (pExistedConsumer && taosArrayGetSize(pConsumerNew->rebNewTopics) == 0 && taosArrayGetSize(pConsumerNew->rebRemovedTopics) == 0) { /*if (taosArrayGetSize(pConsumerNew->assignedTopics) == 0) {*/ /*pConsumerNew->updateType = */ @@ -708,10 +709,11 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { _over: mndTransDrop(pTrans); - if (pConsumerOld) { - /*taosRUnLockLatch(&pConsumerOld->lock);*/ - mndReleaseConsumer(pMnode, pConsumerOld); + if (pExistedConsumer) { + /*taosRUnLockLatch(&pExistedConsumer->lock);*/ + mndReleaseConsumer(pMnode, pExistedConsumer); } + if (pConsumerNew) { tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); @@ -821,20 +823,21 @@ static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mDebug("consumer:0x%" PRIx64 " perform delete action, status:%s", pConsumer->consumerId, mndConsumerStatusName(pConsumer->status)); + mDebug("consumer:0x%" PRIx64 " perform delete action, status:%s", pConsumer->consumerId, + mndConsumerStatusName(pConsumer->status)); tDeleteSMqConsumerObj(pConsumer); return 0; } static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { - mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%"PRId64", uptime:%"PRId64, - pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); + mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, + pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); taosWLockLatch(&pOldConsumer->lock); if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ if (taosArrayGetSize(pNewConsumer->rebNewTopics) == 0 && taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0) { pOldConsumer->status = MQ_CONSUMER_STATUS__READY; @@ -856,13 +859,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; } } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);*/ int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/ for (int32_t i = 0; i < sz; i++) { - char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i)); + char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); } @@ -874,12 +877,12 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { - ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0); - ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0); + /*A(taosArrayGetSize(pOldConsumer->currentTopics) == 0);*/ + /*A(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);*/ int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { - char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); + char *topic = taosStrdup(taosArrayGetP(pOldConsumer->assignedTopics, i)); taosArrayPush(pOldConsumer->rebNewTopics, &topic); } @@ -892,11 +895,12 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { - ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); - ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); + /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);*/ + /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);*/ - char *addedTopic = strdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); + char *addedTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); // not exist in current topic + bool existing = false; #if 1 int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics); @@ -922,6 +926,8 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, if (!existing) { taosArrayPush(pOldConsumer->currentTopics, &addedTopic); taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); + } else { + taosMemoryFree(addedTopic); } // set status @@ -948,15 +954,15 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->currentTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { - ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); - ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1); + /*A(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);*/ + /*A(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 1);*/ char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); // not exist in new topic -#if 1 +#if 0 for (int32_t i = 0; i < taosArrayGetSize(pOldConsumer->rebNewTopics); i++) { char *topic = taosArrayGetP(pOldConsumer->rebNewTopics, i); - ASSERT(strcmp(topic, removedTopic) != 0); + A(strcmp(topic, removedTopic) != 0); } #endif @@ -982,7 +988,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } } // must find the topic - ASSERT(i < sz); + /*A(i < sz);*/ // set status int32_t status = pOldConsumer->status; @@ -1039,14 +1045,14 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * } if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { - mDebug("showing consumer:0x%"PRIx64 " no assigned topic, skip", pConsumer->consumerId); + mDebug("showing consumer:0x%" PRIx64 " no assigned topic, skip", pConsumer->consumerId); sdbRelease(pSdb, pConsumer); continue; } taosRLockLatch(&pConsumer->lock); - mDebug("showing consumer:0x%"PRIx64, pConsumer->consumerId); + mDebug("showing consumer:0x%" PRIx64, pConsumer->consumerId); int32_t topicSz = taosArrayGetSize(pConsumer->assignedTopics); bool hasTopic = true; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1fd49211e0ccc84ce6fb4a6fed618cc0b62496c3..3efd8fb24983de5e731f3bb3ea8d35e509fe3763 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -17,6 +17,7 @@ #include "mndDb.h" #include "mndCluster.h" #include "mndDnode.h" +#include "mndIndex.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndSma.h" @@ -30,9 +31,8 @@ #include "systable.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 54 +#define DB_RESERVE_SIZE 46 -static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb); @@ -74,7 +74,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} -static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { +SSdbRaw *mndDbActionEncode(SDbObj *pDb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + DB_RESERVE_SIZE; @@ -127,6 +127,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER) SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) + SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -216,6 +217,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER) SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -259,6 +261,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; + pOld->cfg.numOfVgroups = pNew->cfg.numOfVgroups; pOld->cfg.buffer = pNew->cfg.buffer; pOld->cfg.pageSize = pNew->cfg.pageSize; pOld->cfg.pages = pNew->cfg.pages; @@ -274,6 +277,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.replications = pNew->cfg.replications; pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; + pOld->compactStartTime = pNew->compactStartTime; taosWUnLockLatch(&pOld->lock); return 0; } @@ -1060,6 +1064,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndUserRemoveDb(pMnode, pTrans, pDb->name) != 0) goto _OVER; @@ -1390,7 +1395,63 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, return 0; } -static int32_t mndCompactDb(SMnode *pMnode, SDbObj *pDb) { return 0; } +static int32_t mndSetCompactDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) { + SDbObj dbObj = {0}; + memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.compactStartTime = compactTs; + + SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbFreeRaw(pCommitRaw); + return -1; + } + + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + return 0; +} + +static int32_t mndSetCompactDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, int64_t compactTs) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (mndVgroupInDb(pVgroup, pDb->uid)) { + if (mndBuildCompactVgroupAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + } + + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndCompactDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { + int64_t compactTs = taosGetTimestampMs(); + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "compact-db"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to compact db:%s", pTrans->id, pDb->name); + mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + if (mndSetCompactDbCommitLogs(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER; + if (mndSetCompactDbRedoActions(pMnode, pTrans, pDb, compactTs) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -1414,10 +1475,11 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - code = mndCompactDb(pMnode, pDb); + code = mndCompactDb(pMnode, pReq, pDb); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: - if (code != 0) { + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c033c3eea9cc1e992216f633912425efad7b616a..fb81a764f133f53f786b1d227e6c8d80f25d9c49 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -171,13 +171,17 @@ void tFreeStreamObj(SStreamObj *pStream) { taosArrayDestroy(pLevel); } taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema + if (pStream->tagSchema.nCols > 0) { + taosMemoryFree(pStream->tagSchema.pSchema); + } } SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { SMqVgEp *pVgEpNew = taosMemoryMalloc(sizeof(SMqVgEp)); if (pVgEpNew == NULL) return NULL; pVgEpNew->vgId = pVgEp->vgId; - pVgEpNew->qmsg = strdup(pVgEp->qmsg); + pVgEpNew->qmsg = taosStrdup(pVgEp->qmsg); pVgEpNew->epSet = pVgEp->epSet; return pVgEpNew; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2c99f5ddf650ea47cab548991b48fd88a648e307..5dd1742e5e9b7946af75105f4dbf5923fc4b4c6d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -180,7 +180,9 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER) terrno = 0; - tmsgUpdateDnodeInfo(&pDnode->id, NULL, pDnode->fqdn, &pDnode->port); + if (tmsgUpdateDnodeInfo(&pDnode->id, NULL, pDnode->fqdn, &pDnode->port)) { + mInfo("dnode:%d, endpoint changed", pDnode->id); + } _OVER: if (terrno != 0) { @@ -189,7 +191,7 @@ _OVER: return NULL; } - mTrace("dnode:%d, decode from raw:%p, row:%p", pDnode->id, pRaw, pDnode); + mTrace("dnode:%d, decode from raw:%p, row:%p ep:%s:%u", pDnode->id, pRaw, pDnode, pDnode->fqdn, pDnode->port); return pRow; } diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 7d0f5742f8bd69df9eb2d21836935dffb84875c0..44a7d49fff672e33af43bb892350ee0b5e02debd 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -629,7 +629,7 @@ void mndDumpSdb() { } taosWriteFile(pFile, pCont, contLen); taosWriteFile(pFile, "\n", 1); - taosFsyncFile(pFile); + UNUSED(taosFsyncFile(pFile)); taosCloseFile(&pFile); tjsonDelete(json); taosMemoryFree(pCont); diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 3ccf00251ec199c740ea3e4f52e9c902511a577f..1b46e169618c2759bc509bf5c542d52c738731ac 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -129,7 +129,7 @@ void grantParseParameter() { mError("can't parsed parameter k"); } void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value) {} void grantAdd(EGrantType grant, uint64_t value) {} void grantRestore(EGrantType grant, uint64_t value) {} -int32_t dmProcessGrantReq(SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } +int32_t dmProcessGrantReq(void* pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } #endif diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c new file mode 100644 index 0000000000000000000000000000000000000000..8782fd823f36fdfcad114d4cf2f6bbac5d754bad --- /dev/null +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -0,0 +1,863 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndIndex.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndIndexComm.h" +#include "mndInfoSchema.h" +#include "mndMnode.h" +#include "mndPrivilege.h" +#include "mndScheduler.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndStream.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "parser.h" +#include "tname.h" + +#define TSDB_IDX_VER_NUMBER 1 +#define TSDB_IDX_RESERVE_SIZE 64 + +static SSdbRaw *mndIdxActionEncode(SIdxObj *pSma); +static SSdbRow *mndIdxActionDecode(SSdbRaw *pRaw); +static int32_t mndIdxActionInsert(SSdb *pSdb, SIdxObj *pIdx); +static int32_t mndIdxActionDelete(SSdb *pSdb, SIdxObj *pIdx); +static int32_t mndIdxActionUpdate(SSdb *pSdb, SIdxObj *pOld, SIdxObj *pNew); +static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq); +// static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); +static int32_t mndProcessGetIdxReq(SRpcMsg *pReq); +static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq); +// static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +// static void mndCancelGetNextIdx(SMnode *pMnode, void *pIter); +static void mndDestroyIdxObj(SIdxObj *pIdxObj); + +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb); + +int32_t mndInitIdx(SMnode *pMnode) { + SSdbTable table = { + .sdbType = SDB_IDX, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndIdxActionEncode, + .decodeFp = (SdbDecodeFp)mndIdxActionDecode, + .insertFp = (SdbInsertFp)mndIdxActionInsert, + .updateFp = (SdbUpdateFp)mndIdxActionUpdate, + .deleteFp = (SdbDeleteFp)mndIdxActionDelete, + }; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_INDEX, mndProcessDropIdxReq); + + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_INDEX_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_INDEX_RSP, mndTransProcessRsp); + + // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropIdxReq); + // mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_GET_TABLE_INDEX, mndProcessGetTbIdxReq); + + // type same with sma + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveIdx); + // mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextIdx); + return sdbSetTable(pMnode->pSdb, table); +} + +static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName) { + for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { + if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { + return tag; + } + } + + return -1; +} + +int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, NULL, 0); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} +static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbObj, SIdxObj *pIdx, int32_t *contLen) { + int32_t len = 0; + int32_t ret = 0; + + SDropIndexReq req = {0}; + memcpy(req.colName, pIdx->colName, sizeof(pIdx->colName)); + memcpy(req.stb, pIdx->stb, sizeof(pIdx->stb)); + req.dbUid = pIdx->dbUid; + req.stbUid = pIdx->stbUid; + + mInfo("idx: %s start to build drop index req", pIdx->name); + + len = tSerializeSDropIdxReq(NULL, 0, &req); + if (ret < 0) { + goto _err; + } + + len += sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryCalloc(1, len); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pHead->contLen = htonl(len); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSDropIdxReq(pBuf, len - sizeof(SMsgHead), &req); + *contLen = len; + return pHead; +_err: + + return NULL; +} +int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + int32_t len; + void *pReq = mndBuildDropIdxReq(pMnode, pVgroup, pStb, pIdx, &len); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = len; + action.msgType = TDMT_VND_DROP_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +void mndCleanupIdx(SMnode *pMnode) { + // do nothing + return; +} + +static SSdbRaw *mndIdxActionEncode(SIdxObj *pIdx) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + // int32_t size = + // sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + pSma->sqlLen + pSma->astLen + TSDB_IDX_RESERVE_SIZE; + int32_t size = sizeof(SIdxObj) + TSDB_IDX_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_IDX, TSDB_IDX_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pIdx->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->dstTbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->colName, TSDB_COL_NAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->stbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->dbUid, _OVER) + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_IDX_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) + + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("idx:%s, failed to encode to raw:%p since %s", pIdx->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("idx:%s, encode to raw:%p, row:%p", pIdx->name, pRaw, pIdx); + return pRaw; +} + +static SSdbRow *mndIdxActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + SSdbRow *pRow = NULL; + SIdxObj *pIdx = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != TSDB_IDX_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + pRow = sdbAllocRow(sizeof(SIdxObj)); + if (pRow == NULL) goto _OVER; + + pIdx = sdbGetRowObj(pRow); + if (pIdx == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_GET_BINARY(pRaw, dataPos, pIdx->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->dstTbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->colName, TSDB_COL_NAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->stbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->dbUid, _OVER) + + SDB_GET_RESERVE(pRaw, dataPos, TSDB_IDX_RESERVE_SIZE, _OVER) + + terrno = 0; + +_OVER: + if (terrno != 0) { + taosMemoryFree(pRow); + return NULL; + } + + mTrace("idx:%s, decode from raw:%p, row:%p", pIdx->name, pRaw, pIdx); + return pRow; +} + +static int32_t mndIdxActionInsert(SSdb *pSdb, SIdxObj *pIdx) { + mTrace("idx:%s, perform insert action, row:%p", pIdx->name, pIdx); + return 0; +} + +static int32_t mndIdxActionDelete(SSdb *pSdb, SIdxObj *pIdx) { + mTrace("idx:%s, perform delete action, row:%p", pIdx->name, pIdx); + return 0; +} + +static int32_t mndIdxActionUpdate(SSdb *pSdb, SIdxObj *pOld, SIdxObj *pNew) { + // lock no not + if (strncmp(pOld->colName, pNew->colName, TSDB_COL_NAME_LEN) != 0) { + memcpy(pOld->colName, pNew->colName, sizeof(pNew->colName)); + } + mTrace("idx:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +SIdxObj *mndAcquireIdx(SMnode *pMnode, char *idxName) { + SSdb *pSdb = pMnode->pSdb; + SIdxObj *pIdx = sdbAcquire(pSdb, SDB_IDX, idxName); + if (pIdx == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; + } + return pIdx; +} + +void mndReleaseIdx(SMnode *pMnode, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pIdx); +} + +SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) { + SName name = {0}; + tNameFromString(&name, idxName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + sdbFreeRaw(pRedoRaw); + return -1; + } + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbFreeRaw(pCommitRaw); + return -1; + } + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { +// SStbObj stbObj = {0}; +// taosRLockLatch(&pStb->lock); +// memcpy(&stbObj, pStb, sizeof(SStbObj)); +// taosRUnLockLatch(&pStb->lock); +// stbObj.numOfColumns = 0; +// stbObj.pColumns = NULL; +// stbObj.numOfTags = 0; +// stbObj.pTags = NULL; +// stbObj.numOfFuncs = 0; +// stbObj.pFuncs = NULL; +// stbObj.updateTime = taosGetTimestampMs(); +// stbObj.lock = 0; +// stbObj.tagVer++; + +// SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); +// if (pCommitRaw == NULL) return -1; +// if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; +// if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; +// +// return 0; +//} + +static void mndDestroyIdxObj(SIdxObj *pIdxObj) { + if (pIdxObj) { + // do nothing + } +} + +static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SStbObj *pStb = NULL; + SIdxObj *pIdx = NULL; + + SDbObj *pDb = NULL; + SCreateTagIndexReq createReq = {0}; + + if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("idx:%s start to create", createReq.idxName); + // if (mndCheckCreateIdxReq(&createReq) != 0) { + // goto _OVER; + // } + + pDb = mndAcquireDbByStb(pMnode, createReq.stbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.stbName); + if (pStb == NULL) { + mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName); + goto _OVER; + } + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) { + pIdx = idx.pIdx; + } else { + goto _OVER; + } + if (pIdx != NULL) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { + return terrno; + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("stb:%s, failed to create since %s", createReq.idxName, terrstr()); + } + + mndReleaseStb(pMnode, pStb); + mndReleaseIdx(pMnode, pIdx); + mndReleaseDb(pMnode, pDb); + + return code; +} + +int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) { + // + return 0; +} + +int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SIdxObj *pIdx = NULL; + int32_t cols = 0; + + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + } + SSmaAndTagIter *pIter = pShow->pIter; + int invalid = -1; + while (numOfRows < rows) { + pIter->pIdxIter = sdbFetch(pSdb, SDB_IDX, pIter->pIdxIter, (void **)&pIdx); + if (pIter->pIdxIter == NULL) break; + + if (NULL != pDb && pIdx->dbUid != pDb->uid) { + sdbRelease(pSdb, pIdx); + continue; + } + + cols = 0; + + SName idxName = {0}; + tNameFromString(&idxName, pIdx->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + STR_TO_VARSTR(n1, (char *)tNameGetTableName(&idxName)); + + char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n2, (char *)mndGetDbStr(pIdx->db)); + + SName stbName = {0}; + tNameFromString(&stbName, pIdx->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); + + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n3, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + colDataAppend(pColInfo, numOfRows, (const char *)&invalid, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); + + char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(col, (char *)pIdx->colName); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)col, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tag, (char *)"tag_index"); + colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + + numOfRows++; + sdbRelease(pSdb, pIdx); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfRows += numOfRows; + return numOfRows; +} + +// static void mndCancelGetNextIdx(SMnode *pMnode, void *pIter) { +// SSdb *pSdb = pMnode->pSdb; +// +// sdbCancelFetch(pSdb, pIter); +//} +static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { + // impl + return TSDB_CODE_SUCCESS; +} + +static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName, + int on) { + taosRLockLatch(&pOld->lock); + memcpy(pNew, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); + + pNew->pTags = NULL; + pNew->updateTime = taosGetTimestampMs(); + pNew->lock = 0; + + int32_t tag = mndFindSuperTableTagId(pOld, tagName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + SSchema *pTag = pNew->pTags + tag; + + if (on == 1) { + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } else { + SSCHMEA_SET_IDX_ON(pTag); + } + } else { + if (!IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + } else { + SSCHMEA_SET_IDX_OFF(pTag); + pTag->flags = 0; + } + } + pNew->tagVer++; + + SSdbRaw *pCommitRaw = mndStbActionEncode(pNew); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} +int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + // impl later + int32_t code = -1; + SStbObj newStb = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb-index"); + if (pTrans == NULL) goto _OVER; + + // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + mndTransSetSerial(pTrans); + + if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + + if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; + if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER; + + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + // mndDestoryIdxObj(pIdx); + if (newStb.pTags != NULL) { + taosMemoryFree(newStb.pTags); + taosMemoryFree(newStb.pColumns); + } + mndTransDrop(pTrans); + return code; +} + +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) { + int32_t code = -1; + SIdxObj idxObj = {0}; + memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN); + memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN); + + idxObj.createdTime = taosGetTimestampMs(); + idxObj.uid = mndGenerateUid(req->idxName, strlen(req->idxName)); + idxObj.stbUid = pStb->uid; + idxObj.dbUid = pStb->dbUid; + + int32_t tag = mndFindSuperTableTagId(pStb, req->colName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } else if (tag == 0) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + + col_id_t colId = pStb->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) { + return -1; + } + + SSchema *pTag = pStb->pTags + tag; + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); + + return code; +} + +static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) { + int32_t code = -1; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; + + SStbObj newObj = {0}; + + pStb = mndAcquireStb(pMnode, pIdx->stb); + if (pStb == NULL) goto _OVER; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); + mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + mndTransSetSerial(pTrans); + if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + + if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; + if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + taosMemoryFree(newObj.pTags); + taosMemoryFree(newObj.pColumns); + + mndTransDrop(pTrans); + mndReleaseStb(pMnode, pStb); + return code; +} +int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SIdxObj *pIdx = NULL; + + SDropTagIndexReq req = {0}; + if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + mInfo("idx:%s, start to drop", req.name); + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) { + pIdx = idx.pIdx; + } else { + goto _OVER; + } + if (pIdx == NULL) { + if (req.igNotExists) { + mInfo("idx:%s, not exist, ignore not exist is set", req.name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbByIdx(pMnode, req.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndDropIdx(pMnode, pReq, pDb, pIdx); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("idx:%s, failed to drop since %s", req.name, terrstr()); + } + mndReleaseIdx(pMnode, pIdx); + mndReleaseDb(pMnode, pDb); + return code; +} +static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) { + // do nothing + return 0; +} + +int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->stbUid == pStb->uid) { + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { + sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIdx); + return -1; + } + } + + sdbRelease(pSdb, pIdx); + } + + return 0; +} + +int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->stbUid == pStb->uid && strcasecmp(pIdx->colName, tagName) == 0) { + memcpy((char *)idx, (char *)pIdx, sizeof(SIdxObj)); + sdbRelease(pSdb, pIdx); + return 0; + } + + sdbRelease(pSdb, pIdx); + } + + return -1; +} +int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->dbUid == pDb->uid) { + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { + sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIdx); + return -1; + } + } + + sdbRelease(pSdb, pIdx); + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndIndexCom.c b/source/dnode/mnode/impl/src/mndIndexCom.c new file mode 100644 index 0000000000000000000000000000000000000000..2858d7548eba72439a2ccfb5d6b75598bea0289b --- /dev/null +++ b/source/dnode/mnode/impl/src/mndIndexCom.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndIndex.h" +#include "mndIndexComm.h" +#include "mndSma.h" + +static void *mndGetIdx(SMnode *pMnode, char *name, int type) { + SSdb *pSdb = pMnode->pSdb; + void *pIdx = sdbAcquire(pSdb, type, name); + if (pIdx == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = 0; + } + return pIdx; +} + +int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) { + SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA); + SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX); + + terrno = 0; + + if (pSma == NULL && pIdx == NULL) return 0; + + if (pSma != NULL) { + if (type == SDB_SMA) { + idx->type = SDB_SMA; + idx->pIdx = pSma; + } else { + mndReleaseSma(pMnode, pSma); + terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + return -1; + } + } else { + if (type == SDB_IDX) { + idx->type = SDB_IDX; + idx->pIdx = pIdx; + } else { + mndReleaseIdx(pMnode, pIdx); + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + } + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 854535c82f72bf5ca4c3340355980ecad07ff7df..53a5548b2f87a5fe2e4318ad052f16f7a1d4ccd9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -21,6 +21,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndGrant.h" +#include "mndIndex.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" @@ -132,6 +133,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { } } +#if 0 static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { int32_t contLen = 0; void *pReq = mndBuildCheckpointTickMsg(&contLen, sec); @@ -144,6 +146,7 @@ static void mndStreamCheckpointTick(SMnode *pMnode, int64_t sec) { tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } +#endif static void mndPullupTelem(SMnode *pMnode) { mTrace("pullup telem msg"); @@ -319,7 +322,7 @@ static void mndCleanupTimer(SMnode *pMnode) { } static int32_t mndCreateDir(SMnode *pMnode, const char *path) { - pMnode->path = strdup(path); + pMnode->path = taosStrdup(path); if (pMnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -425,6 +428,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; @@ -669,9 +673,9 @@ _OVER: mGDebug( "msg:%p, type:%s failed to process since %s, mnode restored:%d stopped:%d, sync restored:%d " - "role:%s, redirect numOfEps:%d inUse:%d", + "role:%s, redirect numOfEps:%d inUse:%d, type:%s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(), pMnode->restored, pMnode->stopped, state.restored, - syncStr(state.restored), epSet.numOfEps, epSet.inUse); + syncStr(state.restored), epSet.numOfEps, epSet.inUse, TMSG_INFO(pMsg->msgType)); if (epSet.numOfEps <= 0) return -1; @@ -712,7 +716,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { } else if (code == 0) { mGTrace("msg:%p, successfully processed", pMsg); } else { - mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, + mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, tstrerror(code), pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 448560d496b30f7bdd9abb8e4d13d382de73c91e..aada00296e994dacedb76fd4595222ba8fc1fd62 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -747,7 +747,7 @@ static void mndReloadSyncConfig(SMnode *pMnode) { pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); - tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { cfg.myIndex = cfg.replicaNum; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b5fba58ba8c5cd023ffc56d678bc9f753f1a0acd..39ee443e51e52781f48eea1450ec7b43632c27e8 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -115,13 +115,11 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream if (pStream->fixedSinkVgId == 0) { SDbObj* pDb = mndAcquireDb(pMnode, pStream->targetDb); - ASSERT(pDb); if (pDb->cfg.numOfVgroups > 1) { isShuffle = true; pTask->outputType = TASK_OUTPUT__SHUFFLE_DISPATCH; pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) { - ASSERT(0); return -1; } } @@ -140,9 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); if (pLastLevelTask->nodeId == pVgInfo->vgId) { - ASSERT(pVgInfo->vgId > 0); pVgInfo->taskId = pLastLevelTask->taskId; - ASSERT(pVgInfo->taskId != 0); break; } } @@ -152,7 +148,6 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only - ASSERT(taosArrayGetSize(pArray) == 1); SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; @@ -170,7 +165,6 @@ int32_t mndAssignTaskToVg(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, co plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { - ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -195,7 +189,6 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan, plan->execNode.epSet = pTask->epSet; if (qSubPlanToString(plan, &pTask->exec.qmsg, &msgLen) < 0) { - ASSERT(0); terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } @@ -222,8 +215,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { void* pIter = NULL; SArray* tasks = taosArrayGetP(pStream->tasks, 0); - ASSERT(taosArrayGetSize(pStream->tasks) == 1); - while (1) { SVgObj* pVgroup = NULL; pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); @@ -257,7 +248,10 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->tbSink.stbUid = pStream->targetStbUid; memcpy(pTask->tbSink.stbFullName, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); - ASSERT(pTask->tbSink.pSchemaWrapper); + if (pTask->tbSink.pSchemaWrapper == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } } sdbRelease(pSdb, pVgroup); } @@ -265,7 +259,6 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { } int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { - ASSERT(pStream->fixedSinkVgId != 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0); SStreamTask* pTask = tNewSStreamTask(pStream->uid); if (pTask == NULL) { @@ -275,8 +268,6 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { pTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(tasks, pTask); - ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId); - pTask->nodeId = pStream->fixedSinkVgId; #if 0 SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); @@ -311,13 +302,16 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } int32_t planTotLevel = LIST_LENGTH(pPlan->pSubplans); - ASSERT(planTotLevel <= 2); + pStream->tasks = taosArrayInit(planTotLevel, sizeof(void*)); bool hasExtraSink = false; bool externalTargetDB = strcmp(pStream->sourceDb, pStream->targetDb) != 0; SDbObj* pDbObj = mndAcquireDb(pMnode, pStream->targetDb); - ASSERT(pDbObj != NULL); + if (pDbObj == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } bool multiTarget = pDbObj->cfg.numOfVgroups > 1; sdbRelease(pSdb, pDbObj); @@ -351,7 +345,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_MERGE); + if (plan->subplanType != SUBPLAN_TYPE_MERGE) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } pInnerTask = tNewSStreamTask(pStream->uid); if (pInnerTask == NULL) { @@ -409,7 +406,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 1); SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } void* pIter = NULL; while (1) { @@ -471,9 +471,15 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { taosArrayPush(pStream->tasks, &taskOneLevel); SNodeListNode* inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0); - ASSERT(LIST_LENGTH(inner->pNodeList) == 1); + if (LIST_LENGTH(inner->pNodeList) != 1) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } SSubplan* plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0); - ASSERT(plan->subplanType == SUBPLAN_TYPE_SCAN); + if (plan->subplanType != SUBPLAN_TYPE_SCAN) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } void* pIter = NULL; while (1) { @@ -585,13 +591,12 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib return -1; } } else { - pVgEp->qmsg = strdup(""); + pVgEp->qmsg = taosStrdup(""); } sdbRelease(pSdb, pVgroup); } - ASSERT(taosArrayGetSize(pSub->unassignedVgs) > 0); qDestroyQueryPlan(pPlan); return 0; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 7a8de4099f0a11f313552a6f2629c797fda06f39..48d8e89bfe73e47dd89e75f8c13626ebd3d1ecf4 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -19,6 +19,7 @@ #include "systable.h" #define SHOW_STEP_SIZE 100 +#define SHOW_COLS_STEP_SIZE 4096 static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq); static void mndFreeShowObj(SShowObj *pShow); @@ -76,6 +77,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_TABLE; } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, len) == 0) { type = TSDB_MGMT_TABLE_TAG; + } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, len) == 0) { + type = TSDB_MGMT_TABLE_COL; } else if (strncasecmp(name, TSDB_INS_TABLE_TABLE_DISTRIBUTED, len) == 0) { // type = TSDB_MGMT_TABLE_DIST; } else if (strncasecmp(name, TSDB_INS_TABLE_USERS, len) == 0) { @@ -131,6 +134,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { showObj.pMnode = pMnode; showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); + strncpy(showObj.filterTb, pReq->filterTb, TSDB_TABLE_NAME_LEN); int32_t keepTime = tsShellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); @@ -190,13 +194,15 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t rowsToRead = SHOW_STEP_SIZE; int32_t size = 0; int32_t rowsRead = 0; - + mDebug("mndProcessRetrieveSysTableReq start"); SRetrieveTableReq retrieveReq = {0}; if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + mDebug("mndProcessRetrieveSysTableReq tb:%s", retrieveReq.tb); + if (retrieveReq.showId == 0) { STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { @@ -226,6 +232,9 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } } + if(pShow->type == TSDB_MGMT_TABLE_COL){ // expend capacity for ins_columns + rowsToRead = SHOW_COLS_STEP_SIZE; + } ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; if (retrieveFp == NULL) { mndReleaseShowObj(pShow, false); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 3eee3a40814c30001d60b6133c7597d5e0a4f198..09aef2cd967fb094b553dec38a837e59050c5e44 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -17,6 +17,8 @@ #include "mndSma.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndIndex.h" +#include "mndIndexComm.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPrivilege.h" @@ -43,9 +45,13 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); -static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); static void mndDestroySmaObj(SSmaObj *pSmaObj); +// sma and tag index comm func +static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); +static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter); + int32_t mndInitSma(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_SMA, @@ -58,14 +64,14 @@ int32_t mndInitSma(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateSmaReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropIdxReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_TABLE_INDEX, mndProcessGetTbSmaReq); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveIdx); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelRetrieveIdx); return sdbSetTable(pMnode->pSdb, table); } @@ -459,8 +465,10 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, int32_t contLen = 0; void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); - taosMemoryFreeClear(pSmaReq); - if (pReq == NULL) return -1; + if (pReq == NULL) { + taosMemoryFreeClear(pSmaReq); + return -1; + } action.pCont = pReq; action.contLen = contLen; @@ -468,10 +476,21 @@ static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, action.acceptableCode = TSDB_CODE_VND_ALREADY_EXIST; if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFreeClear(pSmaReq); taosMemoryFree(pReq); return -1; } + action.pCont = pSmaReq; + action.contLen = smaContLen; + action.msgType = TDMT_VND_CREATE_SMA; + action.acceptableCode = TSDB_CODE_TSMA_ALREADY_EXIST; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFreeClear(pSmaReq); + return -1; + } + return 0; } @@ -533,14 +552,14 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea streamObj.sourceDbUid = pDb->uid; streamObj.targetDbUid = pDb->uid; streamObj.version = 1; - streamObj.sql = strdup(pCreate->sql); + streamObj.sql = taosStrdup(pCreate->sql); streamObj.smaId = smaObj.uid; streamObj.watermark = pCreate->watermark; streamObj.deleteMark = pCreate->deleteMark; streamObj.fillHistory = STREAM_FILL_HISTORY_ON; streamObj.trigger = STREAM_TRIGGER_WINDOW_CLOSE; streamObj.triggerParam = pCreate->maxDelay; - streamObj.ast = strdup(smaObj.ast); + streamObj.ast = taosStrdup(smaObj.ast); // check the maxDelay if (streamObj.triggerParam < TSDB_MIN_ROLLUP_MAX_DELAY) { @@ -699,8 +718,13 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; } + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, createReq.name, SDB_SMA, &idx) == 0) { + pSma = idx.pIdx; + } else { + goto _OVER; + } - pSma = mndAcquireSma(pMnode, createReq.name); if (pSma != NULL) { if (createReq.igExists) { mInfo("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); @@ -946,7 +970,12 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { mInfo("sma:%s, start to drop", dropReq.name); - pSma = mndAcquireSma(pMnode, dropReq.name); + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, dropReq.name, SDB_SMA, &idx) == 0) { + pSma = idx.pIdx; + } else { + goto _OVER; + } if (pSma == NULL) { if (dropReq.igNotExists) { mInfo("sma:%s, not exist, ignore not exist is set", dropReq.name); @@ -985,7 +1014,14 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp int32_t code = -1; SSmaObj *pSma = NULL; - pSma = mndAcquireSma(pMnode, indexReq->indexFName); + SSIdx idx = {0}; + if (0 == mndAcquireGlobalIdx(pMnode, indexReq->indexFName, SDB_SMA, &idx)) { + pSma = idx.pIdx; + } else { + *exist = false; + return 0; + } + if (pSma == NULL) { *exist = false; return 0; @@ -1194,10 +1230,10 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pDb = mndAcquireDb(pMnode, pShow->db); if (pDb == NULL) return 0; } - + SSmaAndTagIter *pIter = pShow->pIter; while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); - if (pShow->pIter == NULL) break; + pIter->pSmaIter = sdbFetch(pSdb, SDB_SMA, pIter->pSmaIter, (void **)&pSma); + if (pIter->pSmaIter == NULL) break; if (NULL != pDb && pSma->dbUid != pDb->uid) { sdbRelease(pSdb, pSma); @@ -1234,6 +1270,18 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); + char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(col, (char *)""); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)col, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tag, (char *)"sma_index"); + colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + numOfRows++; sdbRelease(pSdb, pSma); } @@ -1243,7 +1291,30 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc return numOfRows; } -static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); +// sma and tag index comm func +static int32_t mndProcessDropIdxReq(SRpcMsg *pReq) { + int ret = mndProcessDropSmaReq(pReq); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST) { + terrno = 0; + ret = mndProcessDropTagIdxReq(pReq); + } + return ret; +} + +static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + if (pShow->pIter == NULL) { + pShow->pIter = taosMemoryCalloc(1, sizeof(SSmaAndTagIter)); + } + int32_t read = mndRetrieveSma(pReq, pShow, pBlock, rows); + if (read < rows) read += mndRetrieveTagIdx(pReq, pShow, pBlock, rows - read); + return read; +} +static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter) { + SSmaAndTagIter *p = pIter; + if (p != NULL) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, p->pSmaIter); + sdbCancelFetch(pSdb, p->pIdxIter); + } + taosMemoryFree(p); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 35c8b8adfedcb209567e439a39f19636c927f17f..2a369a863a8f6a79356d058992c0717d16c2a557 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -17,6 +17,7 @@ #include "mndStb.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndIndex.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" @@ -44,11 +45,16 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessDropTtltbReq(SRpcMsg *pReq); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); static int32_t mndProcessTableCfgReq(SRpcMsg *pReq); static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *alterOriData, int32_t alterOriDataLen); -static int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t suid, col_id_t colId); +static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, + void *alterOriData, int32_t alterOriDataLen, const SMAlterStbReq *pAlter); + +static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq); +static int32_t mndProcessDropIndexReq(SRpcMsg *pReq); int32_t mndInitStb(SMnode *pMnode) { SSdbTable table = { @@ -71,10 +77,19 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); + // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); + + // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_INDEX, mndProcessDropIndexReq); + // mndSetMsgHandle(pMnode, TDMT_VND_CREATE_INDEX_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_VND_DROP_INDEX_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_COL, mndRetrieveStbCol); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_COL, mndCancelGetNextStb); + return sdbSetTable(pMnode->pSdb, table); } @@ -421,8 +436,8 @@ static FORCE_INLINE int32_t schemaExColIdCompare(const void *colId, const void * return 0; } -static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, - void *alterOriData, int32_t alterOriDataLen) { +void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void *alterOriData, + int32_t alterOriDataLen) { SEncoder encoder = {0}; int32_t contLen; SName name = {0}; @@ -797,6 +812,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat SSchema *pSchema = &pDst->pTags[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; + if (i == 0) { + SSCHMEA_SET_IDX_ON(pSchema); + } memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = pDst->nextColId; pDst->nextColId++; @@ -875,7 +893,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { - if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { + if (strcmp(pStb->pTags[tag].name, tagName) == 0) { return tag; } } @@ -885,7 +903,7 @@ static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagNam static int32_t mndFindSuperTableColumnIndex(const SStbObj *pStb, const char *colName) { for (int32_t col = 0; col < pStb->numOfColumns; col++) { - if (strcasecmp(pStb->pColumns[col].name, colName) == 0) { + if (strcmp(pStb->pColumns[col].name, colName) == 0) { return col; } } @@ -968,9 +986,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { code = 0; goto _OVER; } else if (pStb->uid != createReq.suid) { - mError("stb:%s, already exist while create, input suid:%" PRId64 " not match with exist suid:%" PRId64, - createReq.name, createReq.suid, pStb->uid); - terrno = TSDB_CODE_MND_STABLE_UID_NOT_MATCH; + mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); + code = 0; goto _OVER; } else if (createReq.tagVer > 0 || createReq.colVer > 0) { int32_t tagDelta = createReq.tagVer - pStb->tagVer; @@ -1082,7 +1099,7 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { return 0; } -static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { +int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); if (pNew->pTags == NULL || pNew->pColumns == NULL) { @@ -1344,6 +1361,10 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj pNew->numOfTags--; pNew->tagVer++; + + // if (mndDropIndexByTag(pMnode, pOld, tagName) != 0) { + // return -1; + // } mInfo("stb:%s, start to drop tag %s", pNew->name, tagName); return 0; } @@ -1605,6 +1626,43 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +static int32_t mndSetAlterStbRedoActions2(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, + void *alterOriData, int32_t alterOriDataLen) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, alterOriData, alterOriDataLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { taosRLockLatch(&pStb->lock); @@ -1643,6 +1701,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa SSchema *pSrcSchema = &pStb->pTags[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1675,7 +1734,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, pRsp->ttl = pStb->ttl; pRsp->commentLen = pStb->commentLen; if (pStb->commentLen > 0) { - pRsp->pComment = strdup(pStb->comment); + pRsp->pComment = taosStrdup(pStb->comment); } for (int32_t i = 0; i < pStb->numOfColumns; ++i) { @@ -1884,6 +1943,74 @@ _OVER: return code; } +static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, + void *alterOriData, int32_t alterOriDataLen, const SMAlterStbReq *pAlter) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + if (needRsp) { + void *pCont = NULL; + int32_t contLen = 0; + if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER; + mndTransSetRpcRsp(pTrans, pCont, contLen); + } + + if (pAlter->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + SIdxObj idxObj = {0}; + SField *pField0 = taosArrayGet(pAlter->pFields, 0); + bool exist = false; + if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { + exist = true; + } + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + + if (exist == true) { + if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + } + + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + } else if (pAlter->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_NAME) { + SIdxObj idxObj = {0}; + SField *pField0 = taosArrayGet(pAlter->pFields, 0); + SField *pField1 = taosArrayGet(pAlter->pFields, 1); + const char *oTagName = pField0->name; + const char *nTagName = pField1->name; + bool exist = false; + + if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { + exist = true; + } + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + + if (exist == true) { + memcpy(idxObj.colName, nTagName, strlen(nTagName)); + idxObj.colName[strlen(nTagName)] = 0; + if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + } + + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + } + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { bool needRsp = true; int32_t code = -1; @@ -1897,7 +2024,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; - + bool updateTagIndex = false; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); @@ -1905,9 +2032,11 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p case TSDB_ALTER_TABLE_DROP_TAG: pField0 = taosArrayGet(pAlter->pFields, 0); code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name); + updateTagIndex = true; break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields); + updateTagIndex = true; break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); @@ -1935,7 +2064,11 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p } if (code != 0) goto _OVER; - code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen); + if (updateTagIndex == false) { + code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen); + } else { + code = mndAlterStbAndUpdateTagIdxImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen, pAlter); + } _OVER: taosMemoryFreeClear(stbObj.pTags); @@ -2068,9 +2201,9 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndDropSmasByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; _OVER: @@ -2178,9 +2311,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, return 0; } -static int32_t mndProcessDropTtltbReq(SRpcMsg *pRsp) { - return 0; -} +static int32_t mndProcessDropTtltbReq(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -2495,6 +2626,282 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t } } +// static int32_t mndProcessRetrieveStbReq(SRpcMsg *pReq) { +// SMnode *pMnode = pReq->info.node; +// SShowMgmt *pMgmt = &pMnode->showMgmt; +// SShowObj *pShow = NULL; +// int32_t rowsToRead = SHOW_STEP_SIZE; +// int32_t rowsRead = 0; +// +// SRetrieveTableReq retrieveReq = {0}; +// if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { +// terrno = TSDB_CODE_INVALID_MSG; +// return -1; +// } +// +// SMnode *pMnode = pReq->info.node; +// SSdb *pSdb = pMnode->pSdb; +// int32_t numOfRows = 0; +// SDbObj *pDb = NULL; +// ESdbStatus objStatus = 0; +// +// SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); +// if (pUser == NULL) return 0; +// bool sysinfo = pUser->sysInfo; +// +// // Append the information_schema database into the result. +//// if (!pShow->sysDbRsp) { +//// SDbObj infoschemaDb = {0}; +//// setInformationSchemaDbCfg(pMnode, &infoschemaDb); +//// size_t numOfTables = 0; +//// getVisibleInfosTablesNum(sysinfo, &numOfTables); +//// mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); +//// +//// numOfRows += 1; +//// +//// SDbObj perfschemaDb = {0}; +//// setPerfSchemaDbCfg(pMnode, &perfschemaDb); +//// numOfTables = 0; +//// getPerfDbMeta(NULL, &numOfTables); +//// mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); +//// +//// numOfRows += 1; +//// pShow->sysDbRsp = true; +//// } +// +// SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); +// blockDataEnsureCapacity(p, rowsToRead); +// +// size_t size = 0; +// const SSysTableMeta* pSysDbTableMeta = NULL; +// +// getInfosDbMeta(&pSysDbTableMeta, &size); +// p->info.rows = buildDbColsInfoBlock(sysinfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); +// +// getPerfDbMeta(&pSysDbTableMeta, &size); +// p->info.rows = buildDbColsInfoBlock(sysinfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); +// +// blockDataDestroy(p); +// +// +// while (numOfRows < rowsToRead) { +// pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus, true); +// if (pShow->pIter == NULL) break; +// if (strncmp(retrieveReq.db, pDb->name, strlen(retrieveReq.db)) != 0){ +// continue; +// } +// if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) { +// continue; +// } +// +// while (numOfRows < rowsToRead) { +// pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); +// if (pShow->pIter == NULL) break; +// +// if (pDb != NULL && pStb->dbUid != pDb->uid) { +// sdbRelease(pSdb, pStb); +// continue; +// } +// +// cols = 0; +// +// SName name = {0}; +// char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; +// mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); +// varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); +// +// SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); +// +// char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; +// tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); +// tNameGetDbName(&name, varDataVal(db)); +// varDataSetLen(db, strlen(varDataVal(db))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)db, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->createdTime, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfColumns, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->numOfTags, false); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false); // number of tables +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// if (pStb->commentLen > 0) { +// char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; +// STR_TO_VARSTR(comment, pStb->comment); +// colDataAppend(pColInfo, numOfRows, comment, false); +// } else if (pStb->commentLen == 0) { +// char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; +// STR_TO_VARSTR(comment, ""); +// colDataAppend(pColInfo, numOfRows, comment, false); +// } else { +// colDataSetNULL(pColInfo, numOfRows); +// } +// +// char watermark[64 + VARSTR_HEADER_SIZE] = {0}; +// sprintf(varDataVal(watermark), "%" PRId64 "a,%" PRId64 "a", pStb->watermark[0], pStb->watermark[1]); +// varDataSetLen(watermark, strlen(varDataVal(watermark))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)watermark, false); +// +// char maxDelay[64 + VARSTR_HEADER_SIZE] = {0}; +// sprintf(varDataVal(maxDelay), "%" PRId64 "a,%" PRId64 "a", pStb->maxdelay[0], pStb->maxdelay[1]); +// varDataSetLen(maxDelay, strlen(varDataVal(maxDelay))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); +// +// char rollup[160 + VARSTR_HEADER_SIZE] = {0}; +// int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); +// char *sep = ", "; +// int32_t sepLen = strlen(sep); +// int32_t rollupLen = sizeof(rollup) - VARSTR_HEADER_SIZE - 2; +// for (int32_t i = 0; i < rollupNum; ++i) { +// char *funcName = taosArrayGet(pStb->pFuncs, i); +// if (i) { +// strncat(varDataVal(rollup), sep, rollupLen); +// rollupLen -= sepLen; +// } +// strncat(varDataVal(rollup), funcName, rollupLen); +// rollupLen -= strlen(funcName); +// } +// varDataSetLen(rollup, strlen(varDataVal(rollup))); +// +// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); +// colDataAppend(pColInfo, numOfRows, (const char *)rollup, false); +// +// numOfRows++; +// sdbRelease(pSdb, pStb); +// } +// +// if (pDb != NULL) { +// mndReleaseDb(pMnode, pDb); +// } +// +// sdbRelease(pSdb, pDb); +// } +// +// pShow->numOfRows += numOfRows; +// mndReleaseUser(pMnode, pUser); +// +// +// +// +// +// +// +// +// ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; +// if (retrieveFp == NULL) { +// mndReleaseShowObj(pShow, false); +// terrno = TSDB_CODE_MSG_NOT_PROCESSED; +// mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); +// return -1; +// } +// +// mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); +// if (retrieveReq.user[0] != 0) { +// memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); +// } else { +// memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER) + 1); +// } +// if (retrieveReq.db[0] && mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { +// return -1; +// } +// +// int32_t numOfCols = pShow->pMeta->numOfColumns; +// +// SSDataBlock *pBlock = createDataBlock(); +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData idata = {0}; +// +// SSchema *p = &pShow->pMeta->pSchemas[i]; +// +// idata.info.bytes = p->bytes; +// idata.info.type = p->type; +// idata.info.colId = p->colId; +// blockDataAppendColInfo(pBlock, &idata); +// } +// +// blockDataEnsureCapacity(pBlock, rowsToRead); +// +// if (mndCheckRetrieveFinished(pShow)) { +// mDebug("show:0x%" PRIx64 ", read finished, numOfRows:%d", pShow->id, pShow->numOfRows); +// rowsRead = 0; +// } else { +// rowsRead = (*retrieveFp)(pReq, pShow, pBlock, rowsToRead); +// if (rowsRead < 0) { +// terrno = rowsRead; +// mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); +// mndReleaseShowObj(pShow, true); +// blockDataDestroy(pBlock); +// return -1; +// } +// +// pBlock->info.rows = rowsRead; +// mDebug("show:0x%" PRIx64 ", stop retrieve data, rowsRead:%d numOfRows:%d", pShow->id, rowsRead, pShow->numOfRows); +// } +// +// size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + +// blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); +// +// SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); +// if (pRsp == NULL) { +// mndReleaseShowObj(pShow, false); +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// mError("show:0x%" PRIx64 ", failed to retrieve data since %s", pShow->id, terrstr()); +// blockDataDestroy(pBlock); +// return -1; +// } +// +// pRsp->handle = htobe64(pShow->id); +// +// if (rowsRead > 0) { +// char *pStart = pRsp->data; +// SSchema *ps = pShow->pMeta->pSchemas; +// +// *(int32_t *)pStart = htonl(pShow->pMeta->numOfColumns); +// pStart += sizeof(int32_t); // number of columns +// +// for (int32_t i = 0; i < pShow->pMeta->numOfColumns; ++i) { +// SSysTableSchema *pSchema = (SSysTableSchema *)pStart; +// pSchema->bytes = htonl(ps[i].bytes); +// pSchema->colId = htons(ps[i].colId); +// pSchema->type = ps[i].type; +// +// pStart += sizeof(SSysTableSchema); +// } +// +// int32_t len = blockEncode(pBlock, pStart, pShow->pMeta->numOfColumns); +// } +// +// pRsp->numOfRows = htonl(rowsRead); +// pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision +// pReq->info.rsp = pRsp; +// pReq->info.rspLen = size; +// +// if (rowsRead == 0 || rowsRead < rowsToRead) { +// pRsp->completed = 1; +// mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); +// mndReleaseShowObj(pShow, true); +// } else { +// mDebug("show:0x%" PRIx64 ", retrieve not completed yet", pShow->id); +// mndReleaseShowObj(pShow, false); +// } +// +// blockDataDestroy(pBlock); +// return TSDB_CODE_SUCCESS; +//} + static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -2605,6 +3012,186 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc return numOfRows; } +static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *pSysDbTableMeta, size_t size, + const char *dbName, const char *tbName) { + char tName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char dName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + STR_TO_VARSTR(dName, dbName); + STR_TO_VARSTR(typeName, "SYSTEM_TABLE"); + + for (int32_t i = 0; i < size; ++i) { + const SSysTableMeta *pm = &pSysDbTableMeta[i]; + // if (pm->sysInfo) { + // continue; + // } + if (tbName[0] && strncmp(tbName, pm->name, TSDB_TABLE_NAME_LEN) != 0) { + continue; + } + + STR_TO_VARSTR(tName, pm->name); + + for (int32_t j = 0; j < pm->colNum; j++) { + // table name + SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tName, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dName, false); + + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, typeName, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, pm->schema[j].name); + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, colName, false); + + // col type + int8_t colType = pm->schema[j].type; + pColInfoData = taosArrayGet(p->pDataBlock, 4); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += + sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", (int32_t)(pm->schema[j].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfoData, numOfRows, (char *)colTypeStr, false); + + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); + for (int32_t k = 6; k <= 8; ++k) { + pColInfoData = taosArrayGet(p->pDataBlock, k); + colDataSetNULL(pColInfoData, numOfRows); + } + + numOfRows += 1; + } + } + + return numOfRows; +} + +static int32_t buildSysDbColsInfo(SSDataBlock *p, char *db, char *tb) { + size_t size = 0; + const SSysTableMeta *pSysDbTableMeta = NULL; + + if (db[0] && strncmp(db, TSDB_INFORMATION_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0 && + strncmp(db, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0) { + return p->info.rows; + } + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbColsInfoBlock(p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB, tb); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbColsInfoBlock(p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB, tb); + + return p->info.rows; +} + +static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SStbObj *pStb = NULL; + + int32_t numOfRows = buildSysDbColsInfo(pBlock, pShow->db, pShow->filterTb); + mDebug("mndRetrieveStbCol get system table cols, rows:%d, db:%s", numOfRows, pShow->db); + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return terrno; + } + + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(typeName, "SUPER_TABLE"); + while (numOfRows < rows) { + pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); + if (pShow->pIter == NULL) break; + + if (pDb != NULL && pStb->dbUid != pDb->uid) { + sdbRelease(pSdb, pStb); + continue; + } + + SName name = {0}; + char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); + if (pShow->filterTb[0] && strncmp(pShow->filterTb, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN) != 0) { + sdbRelease(pSdb, pStb); + continue; + } + varDataSetLen(stbName, strlen(&stbName[VARSTR_HEADER_SIZE])); + + mDebug("mndRetrieveStbCol get stable cols, stable name:%s, db:%s", pStb->name, pStb->db); + + char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&name, pStb->db, T_NAME_ACCT | T_NAME_DB); + tNameGetDbName(&name, varDataVal(db)); + varDataSetLen(db, strlen(varDataVal(db))); + + for (int i = 0; i < pStb->numOfColumns; i++) { + int32_t cols = 0; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)db, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, typeName, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, pStb->pColumns[i].name); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, colName, false); + + // col type + int8_t colType = pStb->pColumns[i].type; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)(pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfo, numOfRows, (char *)colTypeStr, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + while (cols < pShow->numOfColumns) { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); + } + numOfRows++; + } + + sdbRelease(pSdb, pStb); + } + + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + } + + pShow->numOfRows += numOfRows; + mDebug("mndRetrieveStbCol success, rows:%d, pShow->numOfRows:%d", numOfRows, pShow->numOfRows); + + return numOfRows; +} + static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); @@ -2620,3 +3207,136 @@ const char *mndGetStbStr(const char *src) { if (posStb == NULL) return posDb; return posStb; } + +static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { + // impl + return TSDB_CODE_SUCCESS; +} + +/*int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *sql, + int32_t len) { + // impl later + int32_t code = 0; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb-index"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + return code; + +_OVER: + mndTransDrop(pTrans); + return code; +} +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *tagIdxReq, SDbObj *pDb, SStbObj *pOld) { + bool needRsp = true; + int32_t code = -1; + SField *pField0 = NULL; + + SStbObj stbObj = {0}; + SStbObj *pNew = &stbObj; + + taosRLockLatch(&pOld->lock); + memcpy(&stbObj, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); + + stbObj.pColumns = NULL; + stbObj.pTags = NULL; + stbObj.updateTime = taosGetTimestampMs(); + stbObj.lock = 0; + + int32_t tag = mndFindSuperTableTagIndex(pOld, tagIdxReq->colName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + SSchema *pTag = pNew->pTags + tag; + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } else { + pTag->flags |= COL_IDX_ON; + } + pNew->tagVer++; + + code = mndAddIndexImpl(pMnode, pReq, pDb, pNew, needRsp, pReq->pCont, pReq->contLen); + + return code; +} +static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SCreateTagIndexReq tagIdxReq = {0}; + + if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &tagIdxReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("stb:%s, start to alter", tagIdxReq.stbName); + + if (mndCheckIndexReq(&tagIdxReq) != TSDB_CODE_SUCCESS) { + goto _OVER; + } + + pDb = mndAcquireDbByStb(pMnode, tagIdxReq.dbFName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, tagIdxReq.stbName); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + goto _OVER; + } + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndAddIndex(pMnode, pReq, &tagIdxReq, pDb, pStb); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { + return terrno; + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("stb:%s, failed to create index since %s", tagIdxReq.stbName, terrstr()); + } + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + return code; +} +static int32_t mndProcessDropIndexReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SDropTagIndexReq dropReq = {0}; + if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + // + return TSDB_CODE_SUCCESS; +_OVER: + return code; +}*/ diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 19be32708f3fdeda6d33de989fbebb6b885214ca..2a05511134ef6ca310e9766a61e23366318da1c7 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -31,7 +31,7 @@ #define MND_STREAM_VER_NUMBER 2 #define MND_STREAM_RESERVE_SIZE 64 -#define MND_STREAM_MAX_NUM 10 +#define MND_STREAM_MAX_NUM 60 static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); @@ -39,7 +39,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); -static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); +// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); /*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); @@ -66,8 +66,8 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); + // mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); + // mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); @@ -318,7 +318,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, } tstrncpy(pObj->targetDb, pTargetDb->name, TSDB_DB_FNAME_LEN); - pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + if (pCreate->createStb == STREAM_CREATE_STABLE_TRUE) { + pObj->targetStbUid = mndGenerateUid(pObj->targetSTbName, TSDB_TABLE_FNAME_LEN); + } else { + pObj->targetStbUid = pCreate->targetStbUid; + } pObj->targetDbUid = pTargetDb->uid; mndReleaseDb(pMnode, pTargetDb); @@ -330,16 +334,46 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // deserialize ast if (nodesStringToNode(pObj->ast, &pAst) < 0) { - /*ASSERT(0);*/ goto FAIL; } // extract output schema from ast if (qExtractResultSchema(pAst, (int32_t *)&pObj->outputSchema.nCols, &pObj->outputSchema.pSchema) != 0) { - /*ASSERT(0);*/ goto FAIL; } + int32_t numOfNULL = taosArrayGetSize(pCreate->fillNullCols); + if (numOfNULL > 0) { + pObj->outputSchema.nCols += numOfNULL; + SSchema *pFullSchema = taosMemoryCalloc(pObj->outputSchema.nCols, sizeof(SSchema)); + if (!pFullSchema) { + goto FAIL; + } + + int32_t nullIndex = 0; + int32_t dataIndex = 0; + for (int16_t i = 0; i < pObj->outputSchema.nCols; i++) { + SColLocation *pos = taosArrayGet(pCreate->fillNullCols, nullIndex); + if (nullIndex >= numOfNULL || i < pos->slotId) { + pFullSchema[i].bytes = pObj->outputSchema.pSchema[dataIndex].bytes; + pFullSchema[i].colId = i + 1; // pObj->outputSchema.pSchema[dataIndex].colId; + pFullSchema[i].flags = pObj->outputSchema.pSchema[dataIndex].flags; + strcpy(pFullSchema[i].name, pObj->outputSchema.pSchema[dataIndex].name); + pFullSchema[i].type = pObj->outputSchema.pSchema[dataIndex].type; + dataIndex++; + } else { + pFullSchema[i].bytes = 0; + pFullSchema[i].colId = pos->colId; + pFullSchema[i].flags = COL_SET_NULL; + memset(pFullSchema[i].name, 0, TSDB_COL_NAME_LEN); + pFullSchema[i].type = pos->type; + nullIndex++; + } + } + taosMemoryFree(pObj->outputSchema.pSchema); + pObj->outputSchema.pSchema = pFullSchema; + } + SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = false, @@ -353,13 +387,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, // using ast and param to build physical plan if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) { - /*ASSERT(0);*/ goto FAIL; } // save physcial plan if (nodesNodeToString((SNode *)pPlan, false, &pObj->physicalPlan, NULL) != 0) { - /*ASSERT(0);*/ goto FAIL; } @@ -367,7 +399,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, if (pCreate->numOfTags) { pObj->tagSchema.pSchema = taosMemoryCalloc(pCreate->numOfTags, sizeof(SSchema)); } - ASSERT(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags)); + /*A(pCreate->numOfTags == taosArrayGetSize(pCreate->pTags));*/ for (int32_t i = 0; i < pCreate->numOfTags; i++) { SField *pField = taosArrayGet(pCreate->pTags, i); pObj->tagSchema.pSchema[i].colId = pObj->outputSchema.nCols + i + 1; @@ -384,9 +416,6 @@ FAIL: } int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { - if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); - } SEncoder encoder; tEncoderInit(&encoder, NULL, 0); tEncodeSStreamTask(&encoder, pTask); @@ -488,14 +517,27 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre pField->type = pStream->outputSchema.pSchema[i].type; pField->bytes = pStream->outputSchema.pSchema[i].bytes; } - createReq.pTags = taosArrayInit_s(sizeof(SField), 1); - // build tags - SField *pField = taosArrayGet(createReq.pTags, 0); - strcpy(pField->name, "group_id"); - pField->type = TSDB_DATA_TYPE_UBIGINT; - pField->flags = 0; - pField->bytes = 8; + if (pStream->tagSchema.nCols == 0) { + createReq.numOfTags = 1; + createReq.pTags = taosArrayInit_s(sizeof(SField), 1); + // build tags + SField *pField = taosArrayGet(createReq.pTags, 0); + strcpy(pField->name, "group_id"); + pField->type = TSDB_DATA_TYPE_UBIGINT; + pField->flags = 0; + pField->bytes = 8; + } else { + createReq.numOfTags = pStream->tagSchema.nCols; + createReq.pTags = taosArrayInit_s(sizeof(SField), createReq.numOfTags); + for (int32_t i = 0; i < createReq.numOfTags; i++) { + SField *pField = taosArrayGet(createReq.pTags, i); + pField->bytes = pStream->tagSchema.pSchema[i].bytes; + pField->flags = pStream->tagSchema.pSchema[i].flags; + pField->type = pStream->tagSchema.pSchema[i].type; + tstrncpy(pField->name, pStream->tagSchema.pSchema[i].name, TSDB_COL_NAME_LEN); + } + } if (mndCheckCreateStbReq(&createReq) != 0) { goto _OVER; @@ -550,8 +592,6 @@ _OVER: } static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { - ASSERT(pTask->nodeId != 0); - // vnode /*if (pTask->nodeId > 0) {*/ SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); @@ -640,7 +680,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { if (numOfStream > MND_STREAM_MAX_NUM) { - mError("too many streams, no more than 10 for each database"); + mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; goto _OVER; } @@ -652,10 +692,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { } sdbRelease(pMnode->pSdb, pStream); if (numOfStream > MND_STREAM_MAX_NUM) { - mError("too many streams, no more than 10 for each database"); + mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; goto _OVER; } + + if (pStream->targetStbUid == streamObj.targetStbUid) { + mError("Cannot write the same stable as other stream:%s", pStream->name); + terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; + goto _OVER; + } } } @@ -677,10 +723,13 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { mInfo("trans:%d, used to create stream:%s", pTrans->id, createStreamReq.name); mndTransSetDbName(pTrans, createStreamReq.sourceDB, streamObj.targetDb); - if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; - + if (mndTrancCheckConflict(pMnode, pTrans) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } // create stb for stream - if (mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { + if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE && + mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) { mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr()); mndTransDrop(pTrans); goto _OVER; @@ -732,6 +781,9 @@ _OVER: tFreeStreamObj(&streamObj); return code; } + +#if 0 + static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -842,10 +894,9 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { int32_t sz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - ASSERT(pTask->nodeId > 0); + /*A(pTask->nodeId > 0);*/ SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->nodeId); if (pVgObj == NULL) { - ASSERT(0); taosRUnLockLatch(&pStream->lock); mndReleaseStream(pMnode, pStream); mndTransDrop(pTrans); @@ -897,6 +948,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { return 0; } +#endif + static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; @@ -905,7 +958,6 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMDropStreamReq dropReq = {0}; if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { - ASSERT(0); terrno = TSDB_CODE_INVALID_MSG; return -1; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 1dc9ae9c4ba0ebd689da8757dd3aa11eb330a2c8..f618b8afaea6642f723534934dfe175079738c67 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -308,7 +308,7 @@ int32_t mndInitSync(SMnode *pMnode) { pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 4b50361c356618c37f7c00e746be1ccd0fab38d8..991f1099a63aee39526b27371d72cd9699ea83fb 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -388,7 +388,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); topicObj.dbUid = pDb->uid; topicObj.version = 1; - topicObj.sql = strdup(pCreate->sql); + topicObj.sql = taosStrdup(pCreate->sql); topicObj.sqlLen = strlen(pCreate->sql) + 1; topicObj.subType = pCreate->subType; topicObj.withMeta = pCreate->withMeta; @@ -400,7 +400,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * return -1; } - topicObj.ast = strdup(pCreate->ast); + topicObj.ast = taosStrdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; qDebugL("topic:%s ast %s", topicObj.name, topicObj.ast); @@ -687,8 +687,8 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mndReleaseConsumer(pMnode, pConsumer); mndReleaseTopic(pMnode, pTopic); terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", dropReq.name, - pConsumer->consumerId, pConsumer->cgroup); + mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", + dropReq.name, pConsumer->consumerId, pConsumer->cgroup); return -1; } } @@ -722,14 +722,9 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { sdbRelease(pSdb, pConsumer); } -#if 0 - if (pTopic->refConsumerCnt != 0) { - mndReleaseTopic(pMnode, pTopic); - terrno = TSDB_CODE_MND_TOPIC_SUBSCRIBED; - mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq, "drop-topic"); if (pTrans == NULL) { @@ -843,8 +838,8 @@ static int32_t mndRetrieveTopic(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl SName n; int32_t cols = 0; - char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; - const char* pName = mndGetDbStr(pTopic->name); + char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE + 5] = {0}; + const char *pName = mndGetDbStr(pTopic->name); STR_TO_VARSTR(topicName, pName); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 28d0af9eb8b6c32c4397975fb1b12e7f40a84ab7..a34dfff4d6ec679e9ddd450df4efa465ed3fc7eb 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -655,6 +655,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict, taosArrayPush(pTrans->pRpcArray, &pReq->info); pTrans->originRpcType = pReq->msgType; } + mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -1008,6 +1009,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname, pTrans->stbname, &pCont, &contLen) != 0) { mndTransSetRpcRsp(pTrans, pCont, contLen); } + } else if (pTrans->originRpcType == TDMT_MND_CREATE_INDEX) { + void *pCont = NULL; + int32_t contLen = 0; + if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname, pTrans->stbname, &pCont, &contLen) != 0) { + mndTransSetRpcRsp(pTrans, pCont, contLen); + } } if (pTrans->rpcRspLen != 0) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 01a59a48ad636cf9873e90f692a313e602788f40..b7bcaf41fdfd28f22f0783e182ec297f330d29ba 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -59,6 +59,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ALTER_HASHRANGE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_DND_DROP_VNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DISABLE_WRITE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg); @@ -355,9 +356,7 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p SReplica *pReplica = &alterReq.replicas[v]; SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } + if (pVgidDnode == NULL) return NULL; pReplica->id = pVgidDnode->id; pReplica->port = pVgidDnode->port; @@ -397,6 +396,57 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p return pReq; } +static void *mndBuildDisableVnodeWriteReq(SMnode *pMnode, SDbObj *pDb, int32_t vgId, int32_t *pContLen) { + SDisableVnodeWriteReq disableReq = { + .vgId = vgId, + .disable = 1, + }; + + mInfo("vgId:%d, build disable vnode write req", vgId); + int32_t contLen = tSerializeSDisableVnodeWriteReq(NULL, 0, &disableReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSDisableVnodeWriteReq(pReq, contLen, &disableReq); + *pContLen = contLen; + return pReq; +} + +static void *mndBuildAlterVnodeHashRangeReq(SMnode *pMnode, SVgObj *pVgroup, int32_t dstVgId, int32_t *pContLen) { + SAlterVnodeHashRangeReq alterReq = { + .srcVgId = pVgroup->vgId, + .dstVgId = dstVgId, + .hashBegin = pVgroup->hashBegin, + .hashEnd = pVgroup->hashEnd, + }; + + mInfo("vgId:%d, build alter vnode hashrange req, dstVgId:%d, hashrange:[%u, %u]", pVgroup->vgId, dstVgId, + pVgroup->hashBegin, pVgroup->hashEnd); + int32_t contLen = tSerializeSAlterVnodeHashRangeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSAlterVnodeHashRangeReq(pReq, contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -478,7 +528,12 @@ SArray *mndBuildDnodesArray(SMnode *pMnode, int32_t exceptDnodeId) { return pArray; } -static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return *dnode1Id >= *dnode2Id ? 1 : 0; } +static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { + if (*dnode1Id == *dnode2Id) { + return 0; + } + return *dnode1Id > *dnode2Id ? 1 : -1; +} static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float ratio) { float totalDnodes = pDnode->numOfVnodes + (float)pDnode->numOfOtherNodes * ratio + additionDnodes; @@ -488,7 +543,10 @@ static float mndGetDnodeScore(SDnodeObj *pDnode, int32_t additionDnodes, float r static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) { float d1Score = mndGetDnodeScore(pDnode1, 0, 0.9); float d2Score = mndGetDnodeScore(pDnode2, 0, 0.9); - return d1Score >= d2Score ? 1 : 0; + if (d1Score == d2Score) { + return 0; + } + return d1Score > d2Score ? 1 : -1; } void mndSortVnodeGid(SVgObj *pVgroup) { @@ -746,6 +804,13 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pVgroup->isTsma, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // if (pDb == NULL || pDb->compactStartTime <= 0) { + // colDataSetNULL(pColInfo, numOfRows); + // } else { + // colDataAppend(pColInfo, numOfRows, (const char *)&pDb->compactStartTime, false); + // } + numOfRows++; sdbRelease(pSdb, pVgroup); } @@ -1029,6 +1094,7 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + mInfo("vgId:%d, build alter vnode confirm req", pVgroup->vgId); int32_t contLen = sizeof(SMsgHead); SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { @@ -1053,7 +1119,25 @@ int32_t mndAddAlterVnodeConfirmAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } -int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { return 0; } +static int32_t mndAddAlterVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup, int32_t dstVgId) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeHashRangeReq(pMnode, pVgroup, dstVgId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_ALTER_HASHRANGE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} int32_t mndAddAlterVnodeConfigAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { STransAction action = {0}; @@ -1099,6 +1183,31 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } +static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int32_t dnodeId) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode == NULL) return -1; + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildDisableVnodeWriteReq(pMnode, pDb, pVgroup->vgId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DISABLE_WRITE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + int32_t mndAddDropVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SVnodeGid *pVgid, bool isRedo) { STransAction action = {0}; @@ -1763,9 +1872,11 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, } static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { - int32_t code = -1; - STrans *pTrans = NULL; - SArray *pArray = mndBuildDnodesArray(pMnode, 0); + int32_t code = -1; + STrans *pTrans = NULL; + SSdbRaw *pRaw = NULL; + SDbObj dbObj = {0}; + SArray *pArray = mndBuildDnodesArray(pMnode, 0); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "split-vgroup"); if (pTrans == NULL) goto _OVER; @@ -1784,23 +1895,20 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj if (mndAddVnodeToVgroup(pMnode, pTrans, &newVg1, pArray) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddCreateVnodeAction(pMnode, pTrans, pDb, &newVg1, &newVg1.vnodeGid[1]) != 0) goto _OVER; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else if (newVg1.replica == 3) { SVnodeGid del1 = {0}; if (mndRemoveVnodeFromVgroup(pMnode, pTrans, &newVg1, pArray, &del1) != 0) goto _OVER; if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg1, &del1, true) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[0].dnodeId) != 0) goto _OVER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[1].dnodeId) != 0) goto _OVER; - if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; } else { goto _OVER; } - mInfo("vgId:%d, vgroup info after adjust replica, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId, - newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId); for (int32_t i = 0; i < newVg1.replica; ++i) { - mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId); + if (mndAddDisableVnodeWriteAction(pMnode, pTrans, pDb, &newVg1, newVg1.vnodeGid[i].dnodeId) != 0) goto _OVER; } + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; SVgObj newVg2 = {0}; memcpy(&newVg2, &newVg1, sizeof(SVgObj)); @@ -1813,15 +1921,25 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj memcpy(&newVg2.vnodeGid[0], &newVg2.vnodeGid[1], sizeof(SVnodeGid)); memset(&newVg2.vnodeGid[1], 0, sizeof(SVnodeGid)); - mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId, - newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId); - mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId, - newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId); + mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg1.vgId, newVg1.replica, + newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId); + for (int32_t i = 0; i < newVg1.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId); + } + mInfo("vgId:%d, vgroup info after split, replica:%d hashrange:[%u, %u] vnode:0 dnode:%d", newVg2.vgId, newVg2.replica, + newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId); + for (int32_t i = 0; i < newVg1.replica; ++i) { + mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId); + } + + int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg1, maxVgId) != 0) goto _OVER; + newVg1.vgId = maxVgId; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg1) != 0) goto _OVER; - if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, pDb, &newVg2) != 0) goto _OVER; + maxVgId++; + if (mndAddAlterVnodeHashRangeAction(pMnode, pTrans, &newVg2, maxVgId) != 0) goto _OVER; + newVg2.vgId = maxVgId; -#if 0 // adjust vgroup replica if (pDb->cfg.replications != newVg1.replica) { if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg1, pArray) != 0) goto _OVER; @@ -1829,38 +1947,38 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj if (pDb->cfg.replications != newVg2.replica) { if (mndBuildAlterVgroupAction(pMnode, pTrans, pDb, pDb, &newVg2, pArray) != 0) goto _OVER; } -#endif - { - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg1); - if (pRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; - } - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - } + pRaw = mndVgroupActionEncode(&newVg1); + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; - { - SSdbRaw *pRaw = mndVgroupActionEncode(&newVg2); - if (pRaw == NULL) return -1; - if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { - sdbFreeRaw(pRaw); - return -1; - } - (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); - } + pRaw = mndVgroupActionEncode(&newVg2); + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; - mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg1.vgId, - newVg1.replica, newVg1.hashBegin, newVg1.hashEnd, newVg1.vnodeGid[0].dnodeId); - for (int32_t i = 0; i < newVg1.replica; ++i) { - mInfo("vgId:%d, vnode:%d dnode:%d", newVg1.vgId, i, newVg1.vnodeGid[i].dnodeId); - } - mInfo("vgId:%d, vgroup info after adjust hash, replica:%d hashBegin:%u hashEnd:%u vnode:0 dnode:%d", newVg2.vgId, - newVg2.replica, newVg2.hashBegin, newVg2.hashEnd, newVg2.vnodeGid[0].dnodeId); - for (int32_t i = 0; i < newVg1.replica; ++i) { - mInfo("vgId:%d, vnode:%d dnode:%d", newVg2.vgId, i, newVg2.vnodeGid[i].dnodeId); - } + pRaw = mndVgroupActionEncode(pVgroup); + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); + pRaw = NULL; + + memcpy(&dbObj, pDb, sizeof(SDbObj)); + if (dbObj.cfg.pRetensions != NULL) { + dbObj.cfg.pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); + if (dbObj.cfg.pRetensions == NULL) goto _OVER; + } + dbObj.vgVersion++; + dbObj.updateTime = taosGetTimestampMs(); + dbObj.cfg.numOfVgroups++; + pRaw = mndDbActionEncode(&dbObj); + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -1868,22 +1986,29 @@ static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj _OVER: taosArrayDestroy(pArray); mndTransDrop(pTrans); + sdbFreeRaw(pRaw); + taosArrayDestroy(dbObj.cfg.pRetensions); return code; } static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; - int32_t vgId = 2; SVgObj *pVgroup = NULL; SDbObj *pDb = NULL; - mInfo("vgId:%d, start to split", vgId); + SSplitVgroupReq req = {0}; + if (tDeserializeSSplitVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("vgId:%d, start to split", req.vgId); if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } - pVgroup = mndAcquireVgroup(pMnode, vgId); + pVgroup = mndAcquireVgroup(pMnode, req.vgId); if (pVgroup == NULL) goto _OVER; pDb = mndAcquireDb(pMnode, pVgroup->dbName); @@ -2083,3 +2208,59 @@ _OVER: } bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; } + +static void *mndBuildCompactVnodeReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen, + int64_t compactTs) { + SCompactVnodeReq compactReq = {0}; + compactReq.dbUid = pDb->uid; + compactReq.compactStartTime = compactTs; + tstrncpy(compactReq.db, pDb->name, TSDB_DB_FNAME_LEN); + + mInfo("vgId:%d, build compact vnode config req", pVgroup->vgId); + int32_t contLen = tSerializeSCompactVnodeReq(NULL, 0, &compactReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSCompactVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &compactReq); + *pContLen = contLen; + return pReq; +} + +static int32_t mndAddCompactVnodeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, + int64_t compactTs) { + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + + int32_t contLen = 0; + void *pReq = mndBuildCompactVnodeReq(pMnode, pDb, pVgroup, &contLen, compactTs); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_COMPACT; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + +int32_t mndBuildCompactVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int64_t compactTs) { + if (mndAddCompactVnodeAction(pMnode, pTrans, pDb, pVgroup, compactTs) != 0) return -1; + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index 89c2b6931a321adf551516a685286149a7b003b9..2d03631a37725dfd22e93a59dcb078d18fd952ee 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -124,7 +124,7 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendUndolog(pTrans, pUndoRaw); sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - char *param = strdup("====> test log <====="); + char *param = taosStrdup("====> test log <====="); mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1); if (pDb != NULL) { @@ -157,7 +157,7 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendUndolog(pTrans, pUndoRaw); sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - char *param = strdup("====> test action <====="); + char *param = taosStrdup("====> test action <====="); mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1); { @@ -229,7 +229,7 @@ class MndTestTrans2 : public ::testing::Test { mndTransAppendUndolog(pTrans, pUndoRaw); sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - char *param = strdup("====> test log <====="); + char *param = taosStrdup("====> test log <====="); mndTransSetCb(pTrans, TRANS_START_FUNC_TEST, TRANS_STOP_FUNC_TEST, param, strlen(param) + 1); int32_t code = mndTransPrepare(pMnode, pTrans); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 5a44e4279ff2456a9c0df4d134ecc6d4e49801dd..8d2cec478c8d2b3f81231122a4c6d14065ae959b 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -147,7 +147,8 @@ typedef enum { SDB_STB = 18, SDB_DB = 19, SDB_FUNC = 20, - SDB_MAX = 21 + SDB_IDX = 21, + SDB_MAX = 22 } ESdbType; typedef struct SSdbRaw { diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 648ccff432dffe3b460a488940de3dcd953a8db8..bb8040da0773e83ba494ff0325d8b8a1ddf93c7b 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -30,9 +30,9 @@ SSdb *sdbInit(SSdbOpt *pOption) { char path[PATH_MAX + 100] = {0}; snprintf(path, sizeof(path), "%s%sdata", pOption->path, TD_DIRSEP); - pSdb->currDir = strdup(path); + pSdb->currDir = taosStrdup(path); snprintf(path, sizeof(path), "%s%stmp", pOption->path, TD_DIRSEP); - pSdb->tmpDir = strdup(path); + pSdb->tmpDir = taosStrdup(path); if (pSdb->currDir == NULL || pSdb->tmpDir == NULL) { sdbCleanup(pSdb); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 2d4b7a1e56d0f8cdec69e1751065301d4ccae50a..c2d27ad713819f43153222626dce945d291c3e15 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -521,7 +521,7 @@ static SSdbIter *sdbCreateIter(SSdb *pSdb) { char name[PATH_MAX + 100] = {0}; snprintf(name, sizeof(name), "%s%ssdb.data.%" PRIu64, pSdb->tmpDir, TD_DIRSEP, (uint64_t)pIter); - pIter->name = strdup(name); + pIter->name = taosStrdup(name); if (pIter->name == NULL) { taosMemoryFree(pIter); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 505dee3d87053d2b406ba5679419f12bdc5d837b..cc3dba2e07d64f68c017d1b6377fc81f99cad435 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -60,6 +60,8 @@ const char *sdbTableName(ESdbType type) { return "db"; case SDB_FUNC: return "func"; + case SDB_IDX: + return "idx"; default: return "undefine"; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 1d2f4da26b8150c7d92344d091421afd151adcde..d4ca81a6a944a849565f3d86aa0f47bcf00bfc66 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -104,7 +104,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pSnode->path = strdup(path); + pSnode->path = taosStrdup(path); if (pSnode->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 1f7a059ffc30623ddbdc442a9cbeb770afb57fff..ea7046886e5e383ca30a009253ea983d70ba264a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -14,6 +14,7 @@ target_sources( "src/vnd/vnodeSvr.c" "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" + "src/vnd/vnodeCompact.c" # meta "src/meta/metaOpen.c" @@ -53,6 +54,7 @@ target_sources( "src/tsdb/tsdbDiskData.c" "src/tsdb/tsdbCompact.c" "src/tsdb/tsdbMergeTree.c" + "src/tsdb/tsdbDataIter.c" # tq "src/tq/tq.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a9652fb2ff05767f439bfe3385633813ecdf73ed..acfaccafe27c21aa6bf4074ae8759a76f088bae7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -50,13 +50,16 @@ extern const SVnodeCfg vnodeCfgDefault; int32_t vnodeInit(int32_t nthreads); void vnodeCleanup(); int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs); -int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs); +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs); void vnodeDestroy(const char *path, STfs *pTfs); SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb); void vnodePreClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode); void vnodeSyncCheckTimeout(SVnode *pVnode); void vnodeClose(SVnode *pVnode); +int32_t vnodeSyncCommit(SVnode *pVnode); +int32_t vnodeBegin(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); @@ -134,6 +137,7 @@ typedef struct SMetaFltParam { int16_t type; void *val; bool reverse; + bool equal; int (*filterFunc)(void *a, void *b, int16_t type); } SMetaFltParam; @@ -152,7 +156,7 @@ typedef struct SMTbCursor SMTbCursor; SMTbCursor *metaOpenTbCursor(SMeta *pMeta); void metaCloseTbCursor(SMTbCursor *pTbCur); -int32_t metaTbCursorNext(SMTbCursor *pTbCur); +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType); int32_t metaTbCursorPrev(SMTbCursor *pTbCur); #endif @@ -179,6 +183,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo *pTableBlockInfo); @@ -225,11 +230,19 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - int64_t ver; - const SSubmitReq *pMsg; - SSubmitBlk *pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; + // const SSubmitReq *pMsg; + // SSubmitBlk *pBlock; + // SSubmitMsgIter msgIter; + // SSubmitBlkIter blkIter; + + int64_t ver; + SPackedData msg2; + + int8_t setMsg; + SSubmitReq2 submit; + int32_t nextBlk; + + int64_t lastBlkUid; SWalReader *pWalReader; @@ -254,11 +267,14 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver); int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); -int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); +int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); +// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 50e3c3a9c16e54a70d2b4f07a4640d87013a7776..c3e8d7ef1dacd2d0c7aaf143f09d3a9de88eec2e 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -44,7 +44,6 @@ typedef struct SRSmaInfoItem SRSmaInfoItem; typedef struct SRSmaFS SRSmaFS; typedef struct SQTaskFile SQTaskFile; typedef struct SQTaskFReader SQTaskFReader; -typedef struct SQTaskFWriter SQTaskFWriter; struct SSmaEnv { SRWLatch lock; @@ -85,22 +84,20 @@ struct STSmaStat { struct SQTaskFile { volatile int32_t nRef; - int32_t padding; + int8_t level; + int64_t suid; int64_t version; int64_t size; + int64_t mtime; }; struct SQTaskFReader { SSma *pSma; + int8_t level; + int64_t suid; int64_t version; TdFilePtr pReadH; }; -struct SQTaskFWriter { - SSma *pSma; - int64_t version; - TdFilePtr pWriteH; - char *fname; -}; struct SRSmaFS { SArray *aQTaskInf; // array of SQTaskFile @@ -212,87 +209,44 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); } +int32_t smaPreClose(SSma *pSma); + // rsma void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version); +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback); void tdRSmaFSClose(SRSmaFS *fs); -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version); -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version); -int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat); -int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile); -int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer); +int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew); +int32_t tdRSmaFSCommit(SSma *pSma); +int32_t tdRSmaFSFinishCommit(SSma *pSma); +int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS); +void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS); +int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize); +int32_t tdRSmaFSRollback(SSma *pSma); +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); -int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName); +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName); +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, + char *outputName); void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName); void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName); static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_INC(pRSmaInfo); - smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + smaTrace("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } static FORCE_INLINE void tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { int32_t ref = T_REF_DEC(pRSmaInfo); - smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); + smaTrace("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } -// smaFileUtil ================ - -#define TD_FILE_HEAD_SIZE 512 - -typedef struct STFInfo STFInfo; -typedef struct STFile STFile; - -struct STFInfo { - // common fields - uint32_t magic; - uint32_t ftype; - uint32_t fver; - int64_t fsize; -}; - -enum { - TD_FTYPE_RSMA_QTASKINFO = 0, -}; - -#if 0 -struct STFile { - uint8_t state; - STFInfo info; - char *fname; - TdFilePtr pFile; -}; - -#define TD_TFILE_PFILE(tf) ((tf)->pFile) -#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) -#define TD_TFILE_FULL_NAME(tf) ((tf)->fname) -#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL) -#define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf)) -#define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL) -#define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s)) - -int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname); -int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType); -int32_t tdOpenTFile(STFile *pTFile, int flags); -int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); -int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); -int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); -int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); -int64_t tdGetTFileSize(STFile *pTFile, int64_t *size); -int32_t tdRemoveTFile(STFile *pTFile); -int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); -int32_t tdUpdateTFileHeader(STFile *pTFile); -void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); -void tdCloseTFile(STFile *pTFile); -void tdDestroyTFile(STFile *pTFile); -#endif - -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, - char *outputName); -void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); +void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, + int8_t level, int64_t version, char *outputName); +void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 828341ddd8666bbb55c2cedce1b34c50f6b3362a..104e0945ba978815e38f17ecf107ff73c2f2905a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -154,7 +154,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp); +// int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); @@ -181,8 +182,9 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) int32_t tqOffsetCommitFile(STqOffsetStore* pStore); // tqSink -// void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data); -void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, + SBatchDeleteReq* deleteReq); +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data); // tqOffset char* tqOffsetBuildFName(const char* path, int32_t fVer); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 702cbea3334e4170f474a3da1435ec646481a57e..44b7e95f9e43d3037c5171e58ab2fc57b3465b06 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -69,6 +69,11 @@ typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SBlkInfo SBlkInfo; +typedef struct STsdbDataIter2 STsdbDataIter2; +typedef struct STsdbFilterInfo STsdbFilterInfo; + +#define TSDBROW_ROW_FMT ((int8_t)0x0) +#define TSDBROW_COL_FMT ((int8_t)0x1) #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -102,26 +107,29 @@ static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { // tsdbUtil.c ============================================================================================== // TSDBROW -#define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) -#define TSDBROW_VERSION(ROW) (((ROW)->type == 0) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) -#define TSDBROW_SVERSION(ROW) TD_ROW_SVER((ROW)->pTSRow) -#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) -#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = 0, .version = (VERSION), .pTSRow = (TSROW)}) -#define tsdbRowFromBlockData(BLOCKDATA, IROW) ((TSDBROW){.type = 1, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) -void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow); +#define TSDBROW_TS(ROW) (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) +#define TSDBROW_VERSION(ROW) \ + (((ROW)->type == TSDBROW_ROW_FMT) ? (ROW)->version : (ROW)->pBlockData->aVersion[(ROW)->iRow]) +#define TSDBROW_SVERSION(ROW) ((ROW)->type == TSDBROW_ROW_FMT ? (ROW)->pTSRow->sver : -1) +#define TSDBROW_KEY(ROW) ((TSDBKEY){.version = TSDBROW_VERSION(ROW), .ts = TSDBROW_TS(ROW)}) +#define tsdbRowFromTSRow(VERSION, TSROW) ((TSDBROW){.type = TSDBROW_ROW_FMT, .version = (VERSION), .pTSRow = (TSROW)}) +#define tsdbRowFromBlockData(BLOCKDATA, IROW) \ + ((TSDBROW){.type = TSDBROW_COL_FMT, .pBlockData = (BLOCKDATA), .iRow = (IROW)}) + +void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); int32_t tsdbRowCmprFn(const void *p1, const void *p2); // STSDBRowIter -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tRowMergerClear(SRowMerger *pMerger); -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow); -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY @@ -146,24 +154,24 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); // SBlockData -#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) -#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) -#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) -#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) - -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear); -int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClear(SBlockData *pBlockData); -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData); -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]); -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); +#define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) +#define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) +#define tBlockDataFirstKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataFirstRow(PBLOCKDATA)) +#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) +#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) + +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData); +int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); +int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -206,10 +214,10 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); -void tsdbMemTableDestroy(SMemTable *pMemTable); +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); -void tsdbRefMemTable(SMemTable *pMemTable); -void tsdbUnrefMemTable(SMemTable *pMemTable); +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); // STbDataIter int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter); @@ -274,6 +282,7 @@ int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *m int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk); int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); +int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); int32_t tsdbReadSttBlockEx(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); // SDelFWriter @@ -288,8 +297,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id); +int32_t tsdbTakeReadSnap(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap); +void tsdbUntakeReadSnap(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); @@ -305,6 +314,26 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); +// tsdbDataIter.c ============================================================================================== +#define TSDB_MEM_TABLE_DATA_ITER 0 +#define TSDB_DATA_FILE_DATA_ITER 1 +#define TSDB_STT_FILE_DATA_ITER 2 +#define TSDB_TOMB_FILE_DATA_ITER 3 + +#define TSDB_FILTER_FLAG_BY_VERSION 0x1 +#define TSDB_FILTER_FLAG_BY_TABLEID 0x2 + +#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn))) +/* open */ +int32_t tsdbOpenDataFileDataIter(SDataFReader *pReader, STsdbDataIter2 **ppIter); +int32_t tsdbOpenSttFileDataIter(SDataFReader *pReader, int32_t iStt, STsdbDataIter2 **ppIter); +int32_t tsdbOpenTombFileDataIter(SDelFReader *pReader, STsdbDataIter2 **ppIter); +/* close */ +void tsdbCloseDataIter2(STsdbDataIter2 *pIter); +/* cmpr */ +int32_t tsdbDataIterCmprFn(const SRBTreeNode *pNode1, const SRBTreeNode *pNode2); +/* next */ +int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo); // structs ======================= struct STsdbFS { @@ -339,10 +368,13 @@ struct SVersionRange { typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; + int8_t flag; // TSDBROW_ROW_FMT for row format, TSDBROW_COL_FMT for col format + int32_t iRow; int64_t version; - STSRow *pTSRow; + void *pData; SMemSkipListNode *forwards[0]; }; + typedef struct SMemSkipList { int64_t size; uint32_t seed; @@ -368,6 +400,8 @@ struct SMemTable { STsdb *pTsdb; SVBufPool *pPool; volatile int32_t nRef; + int64_t minVer; + int64_t maxVer; TSKEY minKey; TSKEY maxKey; int64_t nRow; @@ -380,11 +414,11 @@ struct SMemTable { }; struct TSDBROW { - int8_t type; // 0 for row from tsRow, 1 for row from block data + int8_t type; // TSDBROW_ROW_FMT for row from tsRow, TSDBROW_COL_FMT for row from block data union { struct { int64_t version; - STSRow *pTSRow; + SRow *pTSRow; }; struct { SBlockData *pBlockData; @@ -470,14 +504,14 @@ struct SSttBlk { // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file // (SBlockData){.suid = 0, .uid = uid}: block data for 1 normal table int .last/.data file struct SBlockData { - int64_t suid; // 0 means normal table block data, otherwise child table block data - int64_t uid; // 0 means block data in .last file, otherwise in .data file - int32_t nRow; // number of rows - int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) - int64_t *aVersion; // versions of each row - TSKEY *aTSKEY; // timestamp of each row - int32_t nColData; - SArray *aColData; // SArray + int64_t suid; // 0 means normal table block data, otherwise child table block data + int64_t uid; // 0 means block data in .last file, otherwise in .data file + int32_t nRow; // number of rows + int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) + int64_t *aVersion; // versions of each row + TSKEY *aTSKEY; // timestamp of each row + int32_t nColData; + SColData *aColData; }; struct TABLEID { @@ -569,10 +603,14 @@ struct SDFileSet { }; struct STSDBRowIter { - TSDBROW *pRow; - STSchema *pTSchema; - SColVal colVal; - int32_t i; + TSDBROW *pRow; + union { + SRowIter *pIter; + struct { + int32_t iColData; + SColVal cv; + }; + }; }; struct SRowMerger { STSchema *pTSchema; @@ -598,9 +636,11 @@ struct SDelFWriter { }; struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; + SMemTable *pMem; + SQueryNode *pNode; + SMemTable *pIMem; + SQueryNode *pINode; + STsdbFS fs; }; struct SDataFWriter { @@ -630,6 +670,7 @@ struct SDataFReader { uint8_t *aBuf[3]; }; +// NOTE: do NOT change the order of the fields typedef struct { int64_t suid; int64_t uid; @@ -720,6 +761,9 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); // tsdbCache ============================================================================================== typedef struct SCacheRowsReader { + STsdb *pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; SVnode *pVnode; STSchema *pSchema; uint64_t uid; @@ -744,8 +788,8 @@ typedef struct { int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup); +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb); +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup); int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, LRUHandle **h); int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); @@ -760,6 +804,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +// int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; @@ -801,12 +847,76 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) { } pIter->pRow = &pIter->row; - pIter->pRow->version = pIter->pNode->version; - pIter->pRow->pTSRow = pIter->pNode->pTSRow; + if (pIter->pNode->flag == TSDBROW_ROW_FMT) { + pIter->row = tsdbRowFromTSRow(pIter->pNode->version, pIter->pNode->pData); + } else if (pIter->pNode->flag == TSDBROW_COL_FMT) { + pIter->row = tsdbRowFromBlockData(pIter->pNode->pData, pIter->pNode->iRow); + } else { + ASSERT(0); + } return pIter->pRow; } +int32_t tRowInfoCmprFn(const void *p1, const void *p2); + +typedef struct { + int64_t suid; + int64_t uid; + SDelData delData; +} SDelInfo; + +struct STsdbDataIter2 { + STsdbDataIter2 *next; + SRBTreeNode rbtn; + + int32_t type; + SRowInfo rowInfo; + SDelInfo delInfo; + union { + // TSDB_MEM_TABLE_DATA_ITER + struct { + SMemTable *pMemTable; + } mIter; + + // TSDB_DATA_FILE_DATA_ITER + struct { + SDataFReader *pReader; + SArray *aBlockIdx; // SArray + SMapData mDataBlk; + SBlockData bData; + int32_t iBlockIdx; + int32_t iDataBlk; + int32_t iRow; + } dIter; + + // TSDB_STT_FILE_DATA_ITER + struct { + SDataFReader *pReader; + int32_t iStt; + SArray *aSttBlk; + SBlockData bData; + int32_t iSttBlk; + int32_t iRow; + } sIter; + // TSDB_TOMB_FILE_DATA_ITER + struct { + SDelFReader *pReader; + SArray *aDelIdx; + SArray *aDelData; + int32_t iDelIdx; + int32_t iDelData; + } tIter; + }; +}; + +struct STsdbFilterInfo { + int32_t flag; + int64_t sver; + int64_t ever; + TABLEID tbid; +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index e075d8a6ca0af9da2a0ca7f12eeb834599fce3e5..88cd1d99e1d9fda6dc28479c4c500220ca37ab96 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -61,10 +61,19 @@ struct SVBufPoolNode { }; struct SVBufPool { - SVBufPool* next; + SVBufPool* freeNext; + SVBufPool* recycleNext; + SVBufPool* recyclePrev; + + // query handle list + TdThreadMutex mutex; + int32_t nQuery; + SQueryNode qList; + SVnode* pVnode; - TdThreadSpinlock* lock; + int32_t id; volatile int32_t nRef; + TdThreadSpinlock* lock; int64_t size; uint8_t* ptr; SVBufPoolNode* pTail; @@ -74,6 +83,8 @@ struct SVBufPool { int32_t vnodeOpenBufPool(SVnode* pVnode); int32_t vnodeCloseBufPool(SVnode* pVnode); void vnodeBufPoolReset(SVBufPool* pPool); +void vnodeBufPoolAddToFreeList(SVBufPool* pPool); +int32_t vnodeBufPoolRecycle(SVBufPool* pPool); // vnodeQuery.c int32_t vnodeQueryOpen(SVnode* pVnode); @@ -89,12 +100,16 @@ int32_t vnodeShouldCommit(SVnode* pVnode); void vnodeUpdCommitSched(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); -int32_t vnodeCommitInfo(const char* dir, const SVnodeInfo* pInfo); +int32_t vnodeCommitInfo(const char* dir); int32_t vnodeLoadInfo(const char* dir, SVnodeInfo* pInfo); int32_t vnodeSyncCommit(SVnode* pVnode); int32_t vnodeAsyncCommit(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode); +// vnodeCompact.c +int32_t vnodeAsyncCompact(SVnode* pVnode); +int32_t vnodeSyncCompact(SVnode* pVnode); + // vnodeSync.c int32_t vnodeSyncOpen(SVnode* pVnode, char* path); int32_t vnodeSyncStart(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index adec0272840870a0498adccd7c67036754752772..93e611e8702a5b8c7ac39e30fdaf008c1134afbc 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,8 @@ typedef struct SRSmaSnapReader SRSmaSnapReader; typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; typedef struct SCommitInfo SCommitInfo; +typedef struct SCompactInfo SCompactInfo; +typedef struct SQueryNode SQueryNode; #define VNODE_META_DIR "meta" #define VNODE_TSDB_DIR "tsdb" @@ -87,16 +89,29 @@ typedef struct SCommitInfo SCommitInfo; #define VNODE_RSMA1_DIR "rsma1" #define VNODE_RSMA2_DIR "rsma2" +#define VNODE_BUFPOOL_SEGMENTS 3 + #define VND_INFO_FNAME "vnode.json" // vnd.h +typedef int32_t (*_query_reseek_func_t)(void* pQHandle); +struct SQueryNode { + SQueryNode* pNext; + SQueryNode** ppNext; + void* pQHandle; + _query_reseek_func_t reseek; +}; + void* vnodeBufPoolMalloc(SVBufPool* pPool, int size); void* vnodeBufPoolMallocAligned(SVBufPool* pPool, int size); void vnodeBufPoolFree(SVBufPool* pPool, void* p); void vnodeBufPoolRef(SVBufPool* pPool); -void vnodeBufPoolUnRef(SVBufPool* pPool); +void vnodeBufPoolUnRef(SVBufPool* pPool, bool proactive); int vnodeDecodeInfo(uint8_t* pData, SVnodeInfo* pInfo); +int32_t vnodeBufPoolRegisterQuery(SVBufPool* pPool, SQueryNode* pQNode); +void vnodeBufPoolDeregisterQuery(SVBufPool* pPool, SQueryNode* pQNode, bool proactive); + // meta typedef struct SMCtbCursor SMCtbCursor; typedef struct SMStbCursor SMStbCursor; @@ -127,6 +142,9 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); +int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); + int64_t metaGetTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock); void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock); @@ -159,13 +177,13 @@ int tsdbClose(STsdb** pTsdb); int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbPrepareCommit(STsdb* pTsdb); int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); +int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbFinishCommit(STsdb* pTsdb); int32_t tsdbRollbackCommit(STsdb* pTsdb); int32_t tsdbDoRetention(STsdb* pTsdb, int64_t now); -int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg); -int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); -int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, - SSubmitBlkRsp* pRsp); +int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); +int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); +int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey); int32_t tsdbSetKeepCfg(STsdb* pTsdb, STsdbCfg* pCfg); @@ -190,7 +208,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); @@ -203,19 +221,12 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqCheckLogInWal(STQ* pTq, int64_t version); -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq); - // sma int32_t smaInit(); void smaCleanUp(); int32_t smaOpen(SVnode* pVnode, int8_t rollback); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); -int32_t smaSyncPreCommit(SSma* pSma); -int32_t smaSyncCommit(SSma* pSma); -int32_t smaSyncPostCommit(SSma* pSma); int32_t smaPrepareAsyncCommit(SSma* pSma); int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); @@ -226,11 +237,10 @@ int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); -int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); +int32_t tdProcessRSmaSubmit(SSma* pSma, int64_t version, void* pReq, void* pMsg, int32_t len, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); -void tdUidStoreDestory(STbUidStore* pStore); void* tdUidStoreFree(STbUidStore* pStore); // SMetaSnapReader ======================================== @@ -277,6 +287,7 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); // SRSmaSnapWriter ======================================== int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter); int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { @@ -338,16 +349,24 @@ typedef struct SVCommitSched { } SVCommitSched; struct SVnode { - char* path; - SVnodeCfg config; - SVState state; - SVStatis statis; - STfs* pTfs; - SMsgCb msgCb; + char* path; + SVnodeCfg config; + SVState state; + SVStatis statis; + STfs* pTfs; + SMsgCb msgCb; + + // Buffer Pool TdThreadMutex mutex; TdThreadCond poolNotEmpty; - SVBufPool* pPool; + SVBufPool* aBufPool[VNODE_BUFPOOL_SEGMENTS]; + SVBufPool* freeList; SVBufPool* inUse; + SVBufPool* onCommit; + SVBufPool* recycleHead; + SVBufPool* recycleTail; + SVBufPool* onRecycle; + SMeta* pMeta; SSma* pSma; STsdb* pTsdb; @@ -426,6 +445,7 @@ enum { struct SSnapDataHdr { int8_t type; + int8_t flag; int64_t index; int64_t size; uint8_t data[]; @@ -437,6 +457,12 @@ struct SCommitInfo { TXN* txn; }; +struct SCompactInfo { + SVnode* pVnode; + int32_t flag; + int64_t commitID; +}; + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 366c072b6f83cf40e404408843504257b2b107f5..05889e47673aa3ed41f5c55693dce0f2d63fa407 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -32,7 +32,7 @@ typedef struct SMetaStbStatsEntry { } SMetaStbStatsEntry; typedef struct STagFilterResEntry { - SList list; // the linked list of md5 digest, extracted from the serialized tag query condition + SList list; // the linked list of md5 digest, extracted from the serialized tag query condition uint32_t hitTimes; // queried times for current super table uint32_t accTime; } STagFilterResEntry; @@ -55,9 +55,9 @@ struct SMetaCache { // query cache struct STagFilterResCache { TdThreadMutex lock; - uint32_t accTimes; - SHashObj* pTableEntry; - SLRUCache* pUidResCache; + uint32_t accTimes; + SHashObj* pTableEntry; + SLRUCache* pUidResCache; } sTagFilterResCache; }; @@ -212,7 +212,7 @@ _exit: int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -223,7 +223,10 @@ int32_t metaCacheUpsert(SMeta* pMeta, SMetaInfo* pInfo) { } if (*ppEntry) { // update - ASSERT(pInfo->suid == (*ppEntry)->info.suid); + if (pInfo->suid != (*ppEntry)->info.suid) { + metaError("meta/cache: suid should be same as the one in cache."); + return TSDB_CODE_FAILED; + } if (pInfo->version > (*ppEntry)->info.version) { (*ppEntry)->info.version = pInfo->version; (*ppEntry)->info.skmVer = pInfo->skmVer; @@ -342,7 +345,7 @@ _exit: int32_t metaStatsCacheUpsert(SMeta* pMeta, SMetaStbStats* pInfo) { int32_t code = 0; - // ASSERT(metaIsWLocked(pMeta)); + // meta is wlocked for calling this func. // search SMetaCache* pCache = pMeta->pCache; @@ -426,12 +429,13 @@ int32_t metaStatsCacheGet(SMeta* pMeta, int64_t uid, SMetaStbStats* pInfo) { return code; } -static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInvalidRes, int32_t keyLen, SLRUCache* pCache, uint64_t suid) { +static int checkAllEntriesInCache(const STagFilterResEntry* pEntry, SArray* pInvalidRes, int32_t keyLen, + SLRUCache* pCache, uint64_t suid) { SListIter iter = {0}; tdListInitIter((SList*)&(pEntry->list), &iter, TD_LIST_FORWARD); SListNode* pNode = NULL; - uint64_t buf[3]; + uint64_t buf[3]; buf[0] = suid; int32_t len = sizeof(uint64_t) * tListLen(buf); @@ -462,7 +466,7 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK *acquireRes = 0; - buf[0] = (uint64_t) pTableMap; + buf[0] = (uint64_t)pTableMap; buf[1] = suid; memcpy(&buf[2], pKey, keyLen); @@ -478,7 +482,11 @@ int32_t metaGetCachedTableUidList(SMeta* pMeta, tb_uid_t suid, const uint8_t* pK // do some book mark work after acquiring the filter result from cache STagFilterResEntry** pEntry = taosHashGet(pTableMap, &suid, sizeof(uint64_t)); - ASSERT(pEntry != NULL); + if (NULL == pEntry) { + metaError("meta/cache: pEntry should not be NULL."); + return TSDB_CODE_FAILED; + } + *acquireRes = 1; const char* p = taosLRUCacheValue(pCache, pHandle); @@ -579,7 +587,7 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int // hash table address(8bytes) + suid(8bytes) + MD5 digest(16bytes) uint64_t buf[4] = {0}; - buf[0] = (uint64_t) pTableEntry; + buf[0] = (uint64_t)pTableEntry; buf[1] = suid; memcpy(&buf[2], pKey, keyLen); ASSERT(keyLen == 16); @@ -612,9 +620,9 @@ int32_t metaUidFilterCachePut(SMeta* pMeta, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, buf, sizeof(uint64_t)*2 + keyLen, pPayload, payloadLen, freePayload, NULL, + taosLRUCacheInsert(pCache, buf, sizeof(uint64_t) * 2 + keyLen, pPayload, payloadLen, freePayload, NULL, TAOS_LRU_PRIORITY_LOW); - _end: +_end: taosThreadMutexUnlock(pLock); metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", TD_VID(pMeta->pVnode), suid, @@ -628,7 +636,7 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) { int32_t keyLen = sizeof(uint64_t) * 3; uint64_t p[4] = {0}; - p[0] = (uint64_t) pMeta->pCache->sTagFilterResCache.pTableEntry; + p[0] = (uint64_t)pMeta->pCache->sTagFilterResCache.pTableEntry; p[1] = suid; TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock; diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index 60df66fc01e745642bcf74dcfe9b10d751edaa3f..e50931ac0618252eea58afc0d1a734d252a05d1e 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -51,7 +51,9 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } else if (pME->type == TSDB_TSMA_TABLE) { if (tEncodeTSma(pCoder, pME->smaEntry.tsma) < 0) return -1; } else { - ASSERT(0); + metaError("meta/entry: invalide table type: %" PRId8 " encode failed.", pME->type); + + return -1; } tEndEncode(pCoder); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 550d7b587cd94c132cad65d6d84540395b0c01e9..1d0b11e26a58d047216196bf3fed7b8f13807922 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -361,7 +361,10 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL return -1; } - ASSERT(pTagIdxKey1->type == pTagIdxKey2->type); + if (pTagIdxKey1->type != pTagIdxKey2->type) { + metaError("meta/open: incorrect tag idx type."); + return TSDB_CODE_FAILED; + } // check NULL, NULL is always the smallest if (pTagIdxKey1->isNull && !pTagIdxKey2->isNull) { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 20f789b348f9b731f050c54ae89cb9d1a422b47f..4b1163bb11138f062ca86ee33f7acdc4a57d640e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -19,7 +19,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags) { memset(pReader, 0, sizeof(*pReader)); pReader->flags = flags; pReader->pMeta = pMeta; - if (!(flags & META_READER_NOLOCK)) { + if (pReader->pMeta && !(flags & META_READER_NOLOCK)) { metaRLock(pMeta); } } @@ -152,7 +152,7 @@ bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { } int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { - SMeta *pMeta = pReader->pMeta; + SMeta *pMeta = pReader->pMeta; int64_t version1; // query uid.idx @@ -239,7 +239,6 @@ int metaGetTableSzNameByUid(void *meta, uint64_t uid, char *tbName) { return 0; } - int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { int code = 0; SMetaReader mr = {0}; @@ -311,7 +310,7 @@ void metaCloseTbCursor(SMTbCursor *pTbCur) { } } -int32_t metaTbCursorNext(SMTbCursor *pTbCur) { +int32_t metaTbCursorNext(SMTbCursor *pTbCur, ETableType jumpTableType) { int ret; void *pBuf; STbCfg tbCfg; @@ -325,7 +324,7 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur) { tDecoderClear(&pTbCur->mr.coder); metaGetTableEntryByVersion(&pTbCur->mr, ((SUidIdxVal *)pTbCur->pVal)[0].version, *(tb_uid_t *)pTbCur->pKey); - if (pTbCur->mr.me.type == TSDB_SUPER_TABLE) { + if (pTbCur->mr.me.type == jumpTableType) { continue; } @@ -359,7 +358,6 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur) { return 0; } - SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { void *pData = NULL; int nData = 0; @@ -637,23 +635,14 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { } STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { - // SMetaReader mr = {0}; STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - STSchemaBuilder sb = {0}; - SSchema *pSchema; + SSchema *pSchema = NULL; pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; - tdInitTSchemaBuilder(&sb, pSW->version); - for (int i = 0; i < pSW->nCols; i++) { - pSchema = pSW->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - pTSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); + pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); taosMemoryFree(pSW->pSchema); taosMemoryFree(pSW); @@ -687,7 +676,13 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv goto _exit; } - ASSERT(c); + if (c == 0) { + metaULock(pMeta); + tdbTbcClose(pSkmDbC); + code = TSDB_CODE_FAILED; + metaError("meta/query: incorrect c: %" PRId32 ".", c); + goto _exit; + } if (c < 0) { tdbTbcMoveToPrev(pSkmDbC); @@ -711,7 +706,10 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv } } - ASSERT(sver > 0); + if (ASSERTS(sver > 0, "failed to get table schema version: %d", sver)) { + code = TSDB_CODE_NOT_FOUND; + goto _exit; + } skmDbKey.uid = suid ? suid : uid; skmDbKey.sver = sver; @@ -734,21 +732,11 @@ int32_t metaGetTbTSchemaEx(SMeta *pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sv tdbFree(pData); // convert - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, pSchemaWrapper->version); - for (int i = 0; i < pSchemaWrapper->nCols; i++) { - SSchema *pSchema = pSchemaWrapper->pSchema + i; - tdAddColToSchema(&sb, pSchema->type, pSchema->flags, pSchema->colId, pSchema->bytes); - } - - STSchema *pTSchema = tdGetSchemaFromBuilder(&sb); + STSchema *pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); if (pTSchema == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; } - tdDestroyTSchemaBuilder(&sb); - *ppTSchema = pTSchema; taosMemoryFree(pSchemaWrapper->pSchema); @@ -781,9 +769,7 @@ int64_t metaGetTimeSeriesNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfTimeSeries + pMeta->pVnode->config.vndStats.numOfNTimeSeries; } -int64_t metaGetNtbNum(SMeta *pMeta) { - return pMeta->pVnode->config.vndStats.numOfNTables; -} +int64_t metaGetNtbNum(SMeta *pMeta) { return pMeta->pVnode->config.vndStats.numOfNTables; } typedef struct { SMeta *pMeta; @@ -1122,26 +1108,30 @@ int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) } int32_t valid = 0; - while (1) { + int32_t count = 0; + + static const int8_t TRY_ERROR_LIMIT = 1; + do { void *entryKey = NULL; int32_t nEntryKey = -1; valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, NULL, NULL); if (valid < 0) break; SCtimeIdxKey *p = entryKey; + if (count > TRY_ERROR_LIMIT) break; int32_t cmp = (*param->filterFunc)((void *)&p->ctime, (void *)&pCtimeKey->ctime, param->type); - if (cmp == 0) taosArrayPush(pUids, &p->uid); - - if (param->reverse == false) { - if (cmp == -1) break; - } else if (param->reverse) { - if (cmp == 1) break; + if (cmp == 0) + taosArrayPush(pUids, &p->uid); + else { + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } - valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) break; - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1176,29 +1166,34 @@ int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (tdbTbcMoveTo(pCursor->pCur, pName, strlen(pName) + 1, &cmp) < 0) { goto END; } - bool first = true; int32_t valid = 0; - while (1) { + int32_t count = 0; + + int32_t TRY_ERROR_LIMIT = 1; + do { void *pEntryKey = NULL, *pEntryVal = NULL; int32_t nEntryKey = -1, nEntryVal = 0; valid = tdbTbcGet(pCursor->pCur, (const void **)pEntryKey, &nEntryKey, (const void **)&pEntryVal, &nEntryVal); if (valid < 0) break; + if (count > TRY_ERROR_LIMIT) break; + char *pTableKey = (char *)pEntryKey; cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type); if (cmp == 0) { tb_uid_t tuid = *(tb_uid_t *)pEntryVal; taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // next } else { - break; + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { break; } - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1254,7 +1249,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { pCursor->type = param->type; metaRLock(pMeta); - ret = tdbTbcOpen(pMeta->pCtimeIdx, &pCursor->pCur, NULL); + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); if (ret < 0) { goto END; } @@ -1298,32 +1293,40 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { goto END; } - bool first = true; + int count = 0; int32_t valid = 0; - while (1) { + bool found = false; + + static const int8_t TRY_ERROR_LIMIT = 1; + + /// src: [[suid, cid1, type1]....[suid, cid2, type2]....[suid, cid3, type3]...] + /// target: [suid, cid2, type2] + do { void *entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { - tdbFree(entryVal); break; } + if (count > TRY_ERROR_LIMIT) { + break; + } + STagIdxKey *p = entryKey; if (p == NULL) break; - if (p->type != pCursor->type) { - if (first) { - valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); - if (valid < 0) break; - continue; - } else { + + if (p->type != pCursor->type || p->suid != pCursor->suid || p->cid != pCursor->cid) { + if (found == true) break; + count++; + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { break; + } else { + continue; } } - if (p->suid != pKey->suid) { - break; - } - first = false; + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { // match @@ -1334,17 +1337,18 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); } taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // not match but should continue to iter + found = true; } else { - // not match and no more result - break; + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { break; } - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1387,19 +1391,19 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { isLock = true; } -// if (taosHashGet(tags, &p->uid, sizeof(tb_uid_t)) == NULL) { - void *val = NULL; - int32_t len = 0; - if (metaGetTableTagByUid(pMeta, suid, p->uid, &val, &len, false) == 0) { - p->pTagVal = taosMemoryMalloc(len); - memcpy(p->pTagVal, val, len); - tdbFree(val); - } else { - metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, - p->uid); - } + // if (taosHashGet(tags, &p->uid, sizeof(tb_uid_t)) == NULL) { + void *val = NULL; + int32_t len = 0; + if (metaGetTableTagByUid(pMeta, suid, p->uid, &val, &len, false) == 0) { + p->pTagVal = taosMemoryMalloc(len); + memcpy(p->pTagVal, val, len); + tdbFree(val); + } else { + metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, + p->uid); } -// } + } + // } if (isLock) metaULock(pMeta); return 0; } @@ -1413,7 +1417,8 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { SHashObj *pSepecifiedUidMap = NULL; size_t numOfElems = taosArrayGetSize(pUidTagInfo); if (numOfElems > 0) { - pSepecifiedUidMap = taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + pSepecifiedUidMap = + taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfElems; i++) { STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, i); taosHashPut(pSepecifiedUidMap, &pTagInfo->uid, sizeof(uint64_t), &i, sizeof(int32_t)); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 8b023b7bc50292dbdd90b52fbeb83eab5d625429..67ade45732865893d730f4920179bdc96a9b4925 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -100,7 +100,10 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - ASSERT(pData && nData); + if (!pData || !nData) { + metaError("meta/snap: invalide nData: %" PRId32 " meta snap read failed.", nData); + goto _exit; + } *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); if (*ppData == NULL) { @@ -249,7 +252,7 @@ static void saveSuperTableInfoForChildTable(SMetaEntry* me, SHashObj* suidInfo) return; } STableInfoForChildTable dataTmp = {0}; - dataTmp.tableName = strdup(me->name); + dataTmp.tableName = taosStrdup(me->name); dataTmp.schemaRow = tCloneSSchemaWrapper(&me->stbEntry.schemaRow); dataTmp.tagRow = tCloneSSchemaWrapper(&me->stbEntry.schemaTag); @@ -358,7 +361,11 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t for (int i = 0; i < taosArrayGetSize(ctx->idList); i++) { int64_t* uid = taosArrayGet(ctx->idList, i); SIdInfo* idData = (SIdInfo*)taosHashGet(ctx->idVersion, uid, sizeof(int64_t)); - ASSERT(idData); + if (!idData) { + metaError("meta/snap: null idData"); + return TSDB_CODE_FAILED; + } + idData->index = i; metaDebug("tmqsnap init idVersion uid:%" PRIi64 " version:%" PRIi64 " index:%d", *uid, idData->version, idData->index); @@ -475,7 +482,10 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return TSDB_CODE_FAILED; + } *uid = *uidTmp; ret = MoveToPosition(ctx, idInfo->version, *uidTmp); @@ -509,7 +519,11 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in (ctx->subType == TOPIC_SUB_TYPE__TABLE && me.type == TSDB_CHILD_TABLE && me.ctbEntry.suid == ctx->suid)) { STableInfoForChildTable* data = (STableInfoForChildTable*)taosHashGet(ctx->suidInfo, &me.ctbEntry.suid, sizeof(tb_uid_t)); - ASSERT(data); + if (!data) { + metaError("meta/snap: null data"); + return TSDB_CODE_FAILED; + } + SVCreateTbReq req = {0}; req.type = TSDB_CHILD_TABLE; @@ -530,7 +544,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in } else { SArray* pTagVals = NULL; if (tTagToValArray((const STag*)p, &pTagVals) != 0) { - ASSERT(0); + metaError("meta/snap: tag to val array failed."); + return TSDB_CODE_FAILED; } int16_t nCols = taosArrayGetSize(pTagVals); for (int j = 0; j < nCols; ++j) { @@ -574,7 +589,8 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in ret = buildNormalChildTableInfo(&req, pBuf, contLen); *type = TDMT_VND_CREATE_TABLE; } else { - ASSERT(0); + metaError("meta/snap: invalid topic sub type: %" PRId8 " get meta from snap failed.", ctx->subType); + ret = -1; } tDecoderClear(&dc); @@ -595,7 +611,10 @@ SMetaTableInfo getUidfromSnapShot(SSnapContext* ctx) { int64_t* uidTmp = taosArrayGet(ctx->idList, ctx->index); ctx->index++; SIdInfo* idInfo = (SIdInfo*)taosHashGet(ctx->idVersion, uidTmp, sizeof(tb_uid_t)); - ASSERT(idInfo); + if (!idInfo) { + metaError("meta/snap: null idInfo"); + return result; + } int32_t ret = MoveToPosition(ctx, idInfo->version, *uidTmp); if (ret != 0) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7406f51d4ba06643af9aa8ce563645c7f8636fb2..94aa464354ca4755fa2f8ca876bd4b6feee3b1f6 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -46,7 +46,7 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->suid = 0; pInfo->skmVer = pEntry->ntbEntry.schemaRow.version; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " get entry info failed.", pEntry->type); } } @@ -342,10 +342,23 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); ret = tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = pReq->suid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(ret == 0 && c == 0); + if (!(ret == 0 && c == 0)) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + metaError("meta/table: invalide ret: %" PRId32 " or c: %" PRId32 "alter stb failed.", ret, c); + return -1; + } ret = tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); - ASSERT(ret == 0); + if (ret < 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + + terrno = TSDB_CODE_TDB_STB_NOT_EXIST; + return -1; + } oStbEntry.pBuf = taosMemoryMalloc(nData); memcpy(oStbEntry.pBuf, pData, nData); @@ -381,6 +394,301 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcClose(pUidIdxc); return 0; } +int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c = -2; + tb_uid_t suid = pReq->suid; + + // get super table + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + ret = -1; + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + if (ret < 0) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { + goto _err; + } + + int diffIdx = -1; + for (int i = 0; i < pReq->schemaTag.nCols; i++) { + SSchema *pNew = pReq->schemaTag.pSchema + i; + SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || + strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { + goto _err; + } + if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { + if (diffIdx != -1) goto _err; + diffIdx = i; + } + } + + if (diffIdx == -1 && diffIdx == 0) { + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &pReq->schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = pTagSchema->pSchema + diffIdx; + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (rc < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + } + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schemaRow = pReq->schemaRow; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + + metaWLock(pMeta); + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + // update uid index + metaUpdateUidIdx(pMeta, &nStbEntry); + metaULock(pMeta); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return TSDB_CODE_VND_COL_ALREADY_EXISTS; +} +int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + int ret = 0; + int c = -2; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + + tb_uid_t suid = pReq->stbUid; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + ret = -1; + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + if (ret < 0) { + goto _err; + } + + SSchema *pCol = NULL; + int32_t colId = -1; + for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { + SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { + if (i != 0 || IS_IDX_ON(schema)) { + pCol = schema; + } + break; + } + } + + if (pCol == NULL) { + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (rc < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + } + + // clear idx flag + SSCHMEA_SET_IDX_OFF(pCol); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = oStbEntry.uid; + nStbEntry.name = oStbEntry.name; + + SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); + SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); + + nStbEntry.stbEntry.schemaRow = *row; + nStbEntry.stbEntry.schemaTag = *tag; + nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; + + metaWLock(pMeta); + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + // update uid index + metaUpdateUidIdx(pMeta, &nStbEntry); + metaULock(pMeta); + + tDeleteSSchemaWrapper(tag); + tDeleteSSchemaWrapper(row); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return -1; +} int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; @@ -558,7 +866,8 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME) { ctime = pME->ntbEntry.ctime; ttlDays = pME->ntbEntry.ttlDays; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " build ttl idx key failed.", pME->type); + return; } if (ttlDays <= 0) return; @@ -635,31 +944,38 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); - const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - if (pTagColumn->type == TSDB_DATA_TYPE_JSON) { + + SSchema *pTagColumn = NULL; + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); } else { - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; - - const void *pTagData = NULL; - int32_t nTagData = 0; - - STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - - if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, - &pTagIdxKey, &nTagIdxKey) == 0) { - tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[i]; + if (!IS_IDX_ON(pTagColumn)) continue; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + + if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, + &pTagIdxKey, &nTagIdxKey) == 0) { + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + } + metaDestroyTagIdxKey(pTagIdxKey); } - metaDestroyTagIdxKey(pTagIdxKey); } tDecoderClear(&tdc); } @@ -773,7 +1089,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -783,7 +1103,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " alt tb column failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -792,7 +1118,13 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + tDecoderClear(&dc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb column failed.", ret); + return -1; + } if (entry.type != TSDB_NORMAL_TABLE) { terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; @@ -812,7 +1144,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl if (iCol >= pSchema->nCols) break; pColumn = &pSchema->pSchema[iCol]; - ASSERT(pAlterTbReq->colName); + if (NULL == pAlterTbReq->colName) { + metaError("meta/table: null pAlterTbReq->colName"); + return -1; + } + if (strcmp(pColumn->name, pAlterTbReq->colName) == 0) break; iCol++; } @@ -897,12 +1233,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl entry.version = version; - metaDeleteNcolIdx(pMeta, &oldEntry); - metaUpdateNcolIdx(pMeta, &entry); - // do actual write metaWLock(pMeta); + metaDeleteNcolIdx(pMeta, &oldEntry); + metaUpdateNcolIdx(pMeta, &entry); // save to table db metaSaveToTbDb(pMeta, &entry); @@ -964,7 +1299,12 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -977,7 +1317,14 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA /* get ctbEntry */ tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + metaError("meta/table: invalide c: %" PRId32 " update tb tag val failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); ctbEntry.pBuf = taosMemoryMalloc(nData); @@ -1071,11 +1418,13 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // save to uid.idx metaUpdateUidIdx(pMeta, &ctbEntry); - if (iCol == 0) { - metaUpdateTagIdx(pMeta, &ctbEntry); + metaUpdateTagIdx(pMeta, &ctbEntry); + + if (NULL == ctbEntry.ctbEntry.pTags) { + metaError("meta/table: null tags, update tag val failed."); + goto _err; } - ASSERT(ctbEntry.ctbEntry.pTags); SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag *)(ctbEntry.ctbEntry.pTags))->len, pMeta->txn); @@ -1130,7 +1479,11 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, NULL); tdbTbcMoveTo(pUidIdxc, &uid, sizeof(uid), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } tdbTbcGet(pUidIdxc, NULL, NULL, &pData, &nData); oversion = ((SUidIdxVal *)pData)[0].version; @@ -1140,7 +1493,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p tdbTbcOpen(pMeta->pTbDb, &pTbDbc, NULL); tdbTbcMoveTo(pTbDbc, &((STbDbKey){.uid = uid, .version = oversion}), sizeof(STbDbKey), &c); - ASSERT(c == 0); + if (c != 0) { + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide c: %" PRId32 " update tb options failed.", c); + return -1; + } + tdbTbcGet(pTbDbc, NULL, NULL, &pData, &nData); // get table entry @@ -1149,7 +1508,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p memcpy(entry.pBuf, pData, nData); tDecoderInit(&dc, entry.pBuf, nData); ret = metaDecodeEntry(&dc, &entry); - ASSERT(ret == 0); + if (ret != 0) { + tDecoderClear(&dc); + tdbTbcClose(pUidIdxc); + tdbTbcClose(pTbDbc); + metaError("meta/table: invalide ret: %" PRId32 " alt tb options failed.", ret); + return -1; + } entry.version = version; metaWLock(pMeta); @@ -1188,6 +1553,243 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p return 0; } +static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t uid, suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + return -1; + } + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + suid = ((SUidIdxVal *)pVal)[0].suid; + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + + if (iCol == 0) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) break; + if (((SCtbIdxKey *)pKey)->suid != uid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + goto _err; + } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + metaDestroyTagIdxKey(pTagIdxKey); + } + return 0; + +_err: + // tDecoderClear(&dc1); + // tDecoderClear(&dc2); + // if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + // if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + // tdbTbcClose(pTbDbc); + // tdbTbcClose(pUidIdxc); + return -1; +} + +typedef struct SMetaPair { + void *key; + int nkey; +} SMetaPair; + +static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + return -1; + } + suid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto _err; + } + + // Get targe schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + if (iCol == 0) { + // cannot drop 1th tag index + terrno = -1; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + if (IS_IDX_ON(pCol)) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + + SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); + + TBC *pTagIdxc = NULL; + tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL); + int rc = + tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); + STagIdxKey *pIdxKey = (STagIdxKey *)pKey; + if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + + SMetaPair pair = {.key = pKey, nKey = nKey}; + taosArrayPush(tagIdxList, &pair); + } + tdbTbcClose(pTagIdxc); + + metaWLock(pMeta); + for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { + SMetaPair *pair = taosArrayGet(tagIdxList, i); + tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); + } + metaULock(pMeta); + + taosArrayDestroy(tagIdxList); + + // set pCol->flags; INDEX_ON + return 0; +_err: + return -1; +} + int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: @@ -1199,6 +1801,10 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta return metaUpdateTableTagVal(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_OPTIONS: return metaUpdateTableOptions(pMeta, version, pReq); + case TSDB_ALTER_TABLE_ADD_TAG_INDEX: + return metaAddTagIndex(pMeta, version, pReq); + case TSDB_ALTER_TABLE_DROP_TAG_INDEX: + return metaDropTagIndex(pMeta, version, pReq); default: terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; return -1; @@ -1363,36 +1969,43 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + STagVal tagVal = {.cid = pTagColumn->colId}; - STagVal tagVal = {.cid = pTagColumn->colId}; - if (pTagColumn->type != TSDB_DATA_TYPE_JSON) { - tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - // pTagData = pCtbEntry->ctbEntry.pTags; - // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; pTagData = pCtbEntry->ctbEntry.pTags; nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); goto end; - } - if (pTagData != NULL) { - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; + } else { + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &pTagSchema->pSchema[i]; + if (i != 0 && !IS_IDX_ON(pTagColumn)) continue; + + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + + if (pTagData != NULL) { + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { + ret = -1; + goto end; + } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + } + metaDestroyTagIdxKey(pTagIdxKey); } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } end: - metaDestroyTagIdxKey(pTagIdxKey); + // metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); tdbFree(pData); return ret; @@ -1411,7 +2024,8 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { } else if (pME->type == TSDB_NORMAL_TABLE) { pSW = &pME->ntbEntry.schemaRow; } else { - ASSERT(0); + metaError("meta/table: invalide table type: %" PRId8 " save skm db failed.", pME->type); + return TSDB_CODE_FAILED; } skmDbKey.uid = pME->uid; diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 20db35e5b561b559bbf5a0145b95bae0ec714d08..51011ef7916a7f5b4173418a6d67186a8ff7869f 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -17,41 +17,18 @@ extern SSmaMgmt smaMgmt; -#if 0 -static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); -#endif -static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); -#if 0 -/** - * @brief Only applicable to Rollup SMA - * - * @param pSma - * @return int32_t - */ -int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaSyncPreCommitImpl(pSma); } - /** - * @brief Only applicable to Rollup SMA + * @brief only applicable to Rollup SMA * * @param pSma * @return int32_t */ -int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } - -/** - * @brief Only applicable to Rollup SMA - * - * @param pSma - * @return int32_t - */ -int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } -#endif +int32_t smaPreClose(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma, false); } /** * @brief async commit, only applicable to Rollup SMA @@ -59,7 +36,7 @@ int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(p * @param pSma * @return int32_t */ -int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } +int32_t smaPrepareAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma, true); } /** * @brief async commit, only applicable to Rollup SMA @@ -128,166 +105,24 @@ _exit: int32_t smaFinishCommit(SSma *pSma) { int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; + code = tdRSmaFSFinishCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + if (VND_RSMA1(pVnode) && (code = tsdbFinishCommit(VND_RSMA1(pVnode))) < 0) { - smaError("vgId:%d, failed to finish commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } if (VND_RSMA2(pVnode) && (code = tsdbFinishCommit(VND_RSMA2(pVnode))) < 0) { - smaError("vgId:%d, failed to finish commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - terrno = code; - return code; -} - -#if 0 -/** - * @brief pre-commit for rollup sma(sync commit). - * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. - * 2) wait for all triggered fetch tasks to finish - * 3) perform persist task for qTaskInfo - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { - return TSDB_CODE_SUCCESS; - } - - SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); - SRSmaStat *pRSmaStat = SMA_STAT_RSMA(pStat); - - // step 1: set rsma stat paused - atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); - - // step 2: wait for all triggered fetch tasks to finish - int32_t nLoops = 0; - while (1) { - if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma fetch tasks are all finished", SMA_VID(pSma)); - break; - } else { - smaDebug("vgId:%d, rsma fetch tasks are not all finished yet", SMA_VID(pSma)); - } - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; - } - } - - // step 3: perform persist task for qTaskInfo - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); - - smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief commit for rollup sma - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { -#if 0 - SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); - if (!pSmaEnv) { - return TSDB_CODE_SUCCESS; - } -#endif - return TSDB_CODE_SUCCESS; -} -#endif - -// SQTaskFile ====================================================== - -/** - * @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be - * multiple qtaskinfo files supporting snapshot replication. - * - * @param pSma - * @param pStat - * @return int32_t - */ -static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { - SVnode *pVnode = pSma->pVnode; - SRSmaFS *pFS = RSMA_FS(pStat); - int64_t committed = pStat->commitAppliedVer; - int64_t fsMaxVer = -1; - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - - taosWLockLatch(RSMA_FS_LOCK(pStat)); - - for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) { - SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i); - int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1); - if ((oldVal <= 1) && (pTaskF->version < committed)) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosRemoveFile(qTaskInfoFullName) < 0) { - smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, - qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno))); - } else { - smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, - qTaskInfoFullName); - } - taosArrayRemove(pFS->aQTaskInf, i); - continue; - } - ++i; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - - if (taosArrayGetSize(pFS->aQTaskInf) > 0) { - fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version; - } - - if (fsMaxVer < committed) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosCheckExistFile(qTaskInfoFullName)) { - SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; - if (!taosArrayPush(pFS->aQTaskInf, &qFile)) { - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } else { - smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode), - committed, fsMaxVer); - } - - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); - return TSDB_CODE_SUCCESS; -} - -#if 0 -/** - * @brief post-commit for rollup sma - * 1) clean up the outdated qtaskinfo files - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { - SVnode *pVnode = pSma->pVnode; - if (!VND_IS_RSMA(pVnode)) { - return TSDB_CODE_SUCCESS; - } - - SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - - tdUpdateQTaskInfoFiles(pSma, pRSmaStat); - - return TSDB_CODE_SUCCESS; + return code; } -#endif /** * @brief Rsma async commit implementation(only do some necessary light weighted task) @@ -295,12 +130,16 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { * 2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) * * @param pSma + * @param isCommit * @return int32_t */ -static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { - return TSDB_CODE_SUCCESS; + return code; } SSmaStat *pStat = SMA_ENV_STAT(pEnv); @@ -309,24 +148,30 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // step 1: set rsma stat atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); - while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) { - ++nLoops; - if (nLoops > 1000) { - sched_yield(); - nLoops = 0; + if (isCommit) { + while (atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 1) != 0) { + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } } - } - pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; - ASSERT(pRSmaStat->commitAppliedVer > 0); + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + if (ASSERTS(pRSmaStat->commitAppliedVer >= -1, "commit applied version %" PRIi64 " < -1", + pRSmaStat->commitAppliedVer)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + } // step 2: wait for all triggered fetch tasks to finish nLoops = 0; while (1) { - if (T_REF_VAL_GET(pStat) == 0) { - smaDebug("vgId:%d, rsma commit, fetch tasks are all finished", SMA_VID(pSma)); + if (atomic_load_32(&pRSmaStat->nFetchAll) <= 0) { + smaDebug("vgId:%d, rsma commit:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit); break; } else { - smaDebug("vgId:%d, rsma commit, fetch tasks are not all finished yet", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit); } ++nLoops; if (nLoops > 1000) { @@ -340,7 +185,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { * 1) This is high cost task and should not put in asyncPreCommit originally. * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. */ - smaInfo("vgId:%d, rsma commit, wait for all items to be consumed, TID:%p", SMA_VID(pSma), + smaInfo("vgId:%d, rsma commit:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit, (void *)taosGetSelfPthreadId()); nLoops = 0; while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { @@ -350,10 +195,13 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { nLoops = 0; } } + + if (!isCommit) goto _exit; + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { - return TSDB_CODE_FAILED; - } + code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + TSDB_CHECK_CODE(code, lino, _exit); + smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); #if 0 // consuming task of qTaskInfo clone @@ -361,8 +209,6 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // lock taosWLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(RSMA_INFO_HASH(pRSmaStat)); - void *pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); while (pIter) { @@ -380,10 +226,20 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { // all rsma results are written completely STsdb *pTsdb = NULL; - if ((pTsdb = VND_RSMA1(pSma->pVnode))) tsdbPrepareCommit(pTsdb); - if ((pTsdb = VND_RSMA2(pSma->pVnode))) tsdbPrepareCommit(pTsdb); + if ((pTsdb = VND_RSMA1(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } + if ((pTsdb = VND_RSMA2(pSma->pVnode))) { + code = tsdbPrepareCommit(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + } - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s(%d)", SMA_VID(pSma), __func__, lino, tstrerror(code), isCommit); + } + return code; } /** @@ -394,30 +250,27 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { */ static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma, SCommitInfo *pInfo) { int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { goto _exit; } -#if 0 - SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - // perform persist task for qTaskInfo operator - if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { - return TSDB_CODE_FAILED; - } -#endif + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommit(VND_RSMA1(pVnode), pInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCommit(VND_RSMA2(pVnode), pInfo); + TSDB_CHECK_CODE(code, lino, _exit); - if ((code = tsdbCommit(VND_RSMA1(pVnode), pInfo)) < 0) { - smaError("vgId:%d, failed to commit tsdb rsma1 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; - } - if ((code = tsdbCommit(VND_RSMA2(pVnode), pInfo)) < 0) { - smaError("vgId:%d, failed to commit tsdb rsma2 since %s", TD_VID(pVnode), tstrerror(code)); - goto _exit; - } _exit: - terrno = code; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -477,8 +330,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); } - tdUpdateQTaskInfoFiles(pSma, pRSmaStat); - atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); return TSDB_CODE_SUCCESS; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 886f8d5c07104aae4533116e348c3f0a9d367f86..02766c8076b435e3cdac12885210f533c12e4c6e 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -131,7 +131,7 @@ static int32_t tdNewSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv) { (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), *ppEnv) : atomic_store_ptr(&SMA_RSMA_ENV(pSma), *ppEnv); - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { + if ((terrno = tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma)) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); *ppEnv = NULL; (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_store_ptr(&SMA_TSMA_ENV(pSma), NULL) @@ -193,10 +193,16 @@ static void tRSmaInfoHashFreeNode(void *data) { } static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { - ASSERT(pSmaStat != NULL); + int32_t code = 0; + int32_t lino = 0; + + if (ASSERTS(pSmaStat != NULL, "pSmaStat is NULL")) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + TSDB_CHECK_CODE(code, lino, _exit); + } if (*pSmaStat) { // no lock - return TSDB_CODE_SUCCESS; + return code; // success, return directly } /** @@ -207,8 +213,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (!(*pSmaStat)) { *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat) + sizeof(TdThread) * tsNumOfVnodeRsmaThreads); if (!(*pSmaStat)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (smaType == TSDB_SMA_TYPE_ROLLUP) { @@ -224,7 +230,8 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (refId < 0) { smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } else { smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM); @@ -235,25 +242,30 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (!RSMA_INFO_HASH(pRSmaStat)) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode); if (tdRsmaStartExecutor(pSma) < 0) { - return TSDB_CODE_FAILED; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } - if (!(RSMA_FS(pRSmaStat)->aQTaskInf = taosArrayInit(1, sizeof(SQTaskFile)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } + taosInitRWLatch(RSMA_FS_LOCK(pRSmaStat)); } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } } - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } static void tdDestroyTSmaStat(STSmaStat *pStat) { @@ -342,7 +354,10 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", vid, refId, smaMgmt.rsetId); } } else { - ASSERT(0); + ASSERTS(0, "unknown smaType:%" PRIi8, smaType); + terrno = TSDB_CODE_APP_ERROR; + smaError("%s failed at line %d since %s", __func__, __LINE__, terrstr()); + return -1; } } return 0; @@ -351,7 +366,7 @@ static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdLockSma(SSma *pSma) { int code = taosThreadMutexLock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to lock td since %s", SMA_VID(pSma), strerror(errno)); + smaError("vgId:%d, failed to lock since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); return -1; } @@ -360,12 +375,17 @@ int32_t tdLockSma(SSma *pSma) { } int32_t tdUnLockSma(SSma *pSma) { - ASSERT(SMA_LOCKED(pSma)); + if (ASSERTS(SMA_LOCKED(pSma), "pSma %p is not locked:%d", pSma, pSma->locked)) { + terrno = TSDB_CODE_APP_ERROR; + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), tstrerror(terrno)); + return -1; + } + pSma->locked = false; int code = taosThreadMutexUnlock(&pSma->mutex); if (code != 0) { - smaError("vgId:%d, failed to unlock td since %s", SMA_VID(pSma), strerror(errno)); terrno = TAOS_SYSTEM_ERROR(code); + smaError("vgId:%d, failed to unlock since %s", SMA_VID(pSma), strerror(errno)); return -1; } return 0; diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c index 8db36be741f3187834d4b2f2eca4fa302ca524fe..1211ef940585a44effb47aa7f09ee69d36abf5e1 100644 --- a/source/dnode/vnode/src/sma/smaFS.c +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -17,250 +17,633 @@ // ================================================================================================= -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); +// static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); -static int32_t tdQTaskInfCmprFn2(const void *p1, const void *p2); -/** - * @brief Open RSma FS from qTaskInfo files - * - * @param pSma - * @param version - * @return int32_t - */ -int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { - SVnode *pVnode = pSma->pVnode; - int64_t commitID = pVnode->state.commitID; - SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SRSmaStat *pStat = NULL; - SArray *output = NULL; - terrno = TSDB_CODE_SUCCESS; +static FORCE_INLINE int32_t tPutQTaskF(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; - if (!pEnv) { - return TSDB_CODE_SUCCESS; - } + n += tPutI8(p ? p + n : p, pFile->level); + n += tPutI64v(p ? p + n : p, pFile->size); + n += tPutI64v(p ? p + n : p, pFile->suid); + n += tPutI64v(p ? p + n : p, pFile->version); + n += tPutI64v(p ? p + n : p, pFile->mtime); + + return n; +} - if (tdFetchQTaskInfoFiles(pSma, version, &output) < 0) { - goto _end; +static int32_t tdRSmaFSToBinary(uint8_t *p, SRSmaFS *pFS) { + int32_t n = 0; + uint32_t size = taosArrayGetSize(pFS->aQTaskInf); + + // version + n += tPutI8(p ? p + n : p, 0); + + // SArray + n += tPutU32v(p ? p + n : p, size); + for (uint32_t i = 0; i < size; ++i) { + n += tPutQTaskF(p ? p + n : p, taosArrayGet(pFS->aQTaskInf, i)); } - pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + return n; +} + +int32_t tdRSmaGetQTaskF(uint8_t *p, SQTaskFile *pFile) { + int32_t n = 0; + + n += tGetI8(p + n, &pFile->level); + n += tGetI64v(p + n, &pFile->size); + n += tGetI64v(p + n, &pFile->suid); + n += tGetI64v(p + n, &pFile->version); + n += tGetI64v(p + n, &pFile->mtime); + + return n; +} + +static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, SRSmaFS *pFS) { + int32_t code = 0; + int32_t n = 0; + int8_t version = 0; + + // version + n += tGetI8(pData + n, &version); + + // SArray + taosArrayClear(pFS->aQTaskInf); + uint32_t size = 0; + n += tGetU32v(pData + n, &size); + for (uint32_t i = 0; i < size; ++i) { + SQTaskFile qTaskF = {0}; + + int32_t nt = tdRSmaGetQTaskF(pData + n, &qTaskF); + if (nt < 0) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - int32_t vid = 0; - int64_t version = -1; - sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); - SQTaskFile qTaskFile = {.version = version, .nRef = 1}; - if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { - goto _end; + n += nt; + if (taosArrayPush(pFS->aQTaskInf, &qTaskF) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%d", TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); } -_end: - for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { - void *ptr = taosArrayGetP(output, i); - taosMemoryFreeClear(ptr); + if (ASSERTS(n + sizeof(TSCKSUM) == nData, "n:%d + sizeof(TSCKSUM):%d != nData:%d", n, (int32_t)sizeof(TSCKSUM), + nData)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + +_exit: + return code; +} + +static int32_t tdRSmaSaveFSToFile(SRSmaFS *pFS, const char *fname) { + int32_t code = 0; + int32_t lino = 0; + + // encode to binary + int32_t size = tdRSmaFSToBinary(NULL, pFS) + sizeof(TSCKSUM); + uint8_t *pData = taosMemoryMalloc(size); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + tdRSmaFSToBinary(pData, pFS); + taosCalcChecksumAppend(0, pData, size); + + // save to file + TdFilePtr pFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t n = taosWriteFile(pFD, pData, size); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (taosFsyncFile(pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&pFD); + +_exit: + if (pData) taosMemoryFree(pData); + if (code) { + smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + return code; +} + +static int32_t tdRSmaFSCreate(SRSmaFS *pFS, int32_t size) { + int32_t code = 0; + + pFS->aQTaskInf = taosArrayInit(size, sizeof(SQTaskFile)); + if (pFS->aQTaskInf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - taosArrayDestroy(output); - if (terrno != TSDB_CODE_SUCCESS) { - smaError("vgId:%d, open rsma fs failed since %s", TD_VID(pVnode), terrstr()); - return TSDB_CODE_FAILED; +_exit: + return code; +} + +static void tdRSmaGetCurrentFName(SSma *pSma, char *current, char *current_t) { + SVnode *pVnode = pSma->pVnode; + if (pVnode->pTfs) { + if (current) { + snprintf(current, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT", tfsGetPrimaryPath(pVnode->pTfs), + TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); + } + if (current_t) { + snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%svnode%svnode%d%srsma%sPRESENT.t", tfsGetPrimaryPath(pVnode->pTfs), + TD_DIRSEP, TD_DIRSEP, TD_VID(pVnode), TD_DIRSEP, TD_DIRSEP); + } + } else { +#if 0 + if (current) { + snprintf(current, TSDB_FILENAME_LEN - 1, "%s%sPRESENT", pTsdb->path, TD_DIRSEP); + } + if (current_t) { + snprintf(current_t, TSDB_FILENAME_LEN - 1, "%s%sPRESENT.t", pTsdb->path, TD_DIRSEP); + } +#endif } - return TSDB_CODE_SUCCESS; } -void tdRSmaFSClose(SRSmaFS *fs) { taosArrayDestroy(fs->aQTaskInf); } +static int32_t tdRSmaLoadFSFromFile(const char *fname, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + uint8_t *pData = NULL; + + // load binary + TdFilePtr pFD = taosOpenFile(fname, TD_FILE_READ); + if (pFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size; + if (taosFStatFile(pFD, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + pData = taosMemoryMalloc(size); + if (pData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (taosReadFile(pFD, pData, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (!taosCheckChecksumWhole(pData, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + taosCloseFile(&pFD); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&pFD); + + // decode binary + code = tsdbBinaryToFS(pData, size, pFS); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (pData) taosMemoryFree(pData); + if (code) { + smaError("%s failed at line %d since %s, fname:%s", __func__, lino, tstrerror(code), fname); + } + return code; +} static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { - if (*(int64_t *)p1 < ((SQTaskFile *)p2)->version) { + const SQTaskFile *q1 = (const SQTaskFile *)p1; + const SQTaskFile *q2 = (const SQTaskFile *)p2; + + if (q1->suid < q2->suid) { + return -1; + } else if (q1->suid > q2->suid) { + return 1; + } + + if (q1->level < q2->level) { return -1; - } else if (*(int64_t *)p1 > ((SQTaskFile *)p2)->version) { + } else if (q1->level > q2->level) { + return 1; + } + + if (q1->version < q2->version) { + return -2; + } else if (q1->version > q2->version) { return 1; } + return 0; } -int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - SQTaskFile *pTaskF = NULL; - int32_t oldVal = 0; +static int32_t tdRSmaFSApplyChange(SSma *pSma, SRSmaFS *pFSNew) { + int32_t code = 0; + int32_t lino = 0; + int32_t nRef = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *pFSOld = RSMA_FS(pStat); + int64_t version = pStat->commitAppliedVer; + char fname[TSDB_FILENAME_LEN] = {0}; + + // SQTaskFile + int32_t nNew = taosArrayGetSize(pFSNew->aQTaskInf); + int32_t iNew = 0; + while (iNew < nNew) { + SQTaskFile *pQTaskFNew = TARRAY_GET_ELEM(pFSNew->aQTaskInf, iNew++); + + int32_t idx = taosArraySearchIdx(pFSOld->aQTaskInf, pQTaskFNew, tdQTaskInfCmprFn1, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFSOld->aQTaskInf); + pQTaskFNew->nRef = 1; + } else { + SQTaskFile *pTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); + int32_t c1 = tdQTaskInfCmprFn1(pQTaskFNew, pTaskF); + if (c1 == 0) { + // utilize the item in pFSOld->qQTaskInf, instead of pFSNew + continue; + } else if (c1 < 0) { + // NOTHING TODO + } else { + code = TSDB_CODE_RSMA_FS_UPDATE; + TSDB_CHECK_CODE(code, lino, _exit); + } + } - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if ((pTaskF = taosArraySearch(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ))) { - oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); - ASSERT(oldVal > 0); + if (taosArrayInsert(pFSOld->aQTaskInf, idx, pQTaskFNew) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // remove previous version + while (--idx >= 0) { + SQTaskFile *preTaskF = TARRAY_GET_ELEM(pFSOld->aQTaskInf, idx); + int32_t c2 = tdQTaskInfCmprFn1(preTaskF, pQTaskFNew); + if (c2 == 0) { + code = TSDB_CODE_RSMA_FS_UPDATE; + TSDB_CHECK_CODE(code, lino, _exit); + } else if (c2 != -2) { + break; + } + + nRef = atomic_sub_fetch_32(&preTaskF->nRef, 1); + if (nRef <= 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), preTaskF->suid, preTaskF->level, preTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + (void)taosRemoveFile(fname); + taosArrayRemove(pFSOld->aQTaskInf, idx); + } + } } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return oldVal; + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } -int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - int64_t version = -1; +static int32_t tdRSmaFSScanAndTryFix(SSma *pSma) { + int32_t code = 0; +#if 0 + int32_t lino = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *pFS = RSMA_FS(pStat); + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameVer[TSDB_FILENAME_LEN] = {0}; + + // SArray + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); + + // main.tdb ========= + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fnameVer); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); + + if (taosCheckExistFile(fnameVer)) { + if (taosRenameFile(fnameVer, fname) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, %s:%d succeed to to rename %s to %s", TD_VID(pVnode), __func__, lino, fnameVer, fname); + } else if (taosCheckExistFile(fname)) { + if (taosRemoveFile(fname) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, %s:%d succeed to to remove %s", TD_VID(pVnode), __func__, lino, fname); + } + } - taosRLockLatch(RSMA_FS_LOCK(pStat)); - if (taosArrayGetSize(aQTaskInf) > 0) { - version = ((SQTaskFile *)taosArrayGetLast(aQTaskInf))->version; + { + // remove those invalid files (todo) + // main.tdb-journal.5 // TDB should handle its clear for kill -9 } - taosRUnLockLatch(RSMA_FS_LOCK(pStat)); - return version; + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } +#endif + return code; } -void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { - SVnode *pVnode = pSma->pVnode; - SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; - char qTaskFullName[TSDB_FILENAME_LEN]; - SQTaskFile *pTaskF = NULL; - int32_t idx = -1; +// EXPOSED APIS ==================================================================================== - taosWLockLatch(RSMA_FS_LOCK(pStat)); - if ((idx = taosArraySearchIdx(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { - ASSERT(idx < taosArrayGetSize(aQTaskInf)); - pTaskF = taosArrayGet(aQTaskInf, idx); - if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); - if (taosRemoveFile(qTaskFullName) < 0) { - smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, - tstrerror(TAOS_SYSTEM_ERROR(errno))); +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + // open handle + code = tdRSmaFSCreate(RSMA_FS(pStat), 0); + TSDB_CHECK_CODE(code, lino, _exit); + + // open impl + char current[TSDB_FILENAME_LEN] = {0}; + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, current, current_t); + + if (taosCheckExistFile(current)) { + code = tdRSmaLoadFSFromFile(current, RSMA_FS(pStat)); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosCheckExistFile(current_t)) { + if (rollback) { + code = tdRSmaFSRollback(pSma); + TSDB_CHECK_CODE(code, lino, _exit); } else { - smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); } - taosArrayRemove(aQTaskInf, idx); } + } else { + // 1st time open with empty current/qTaskInfoFile + code = tdRSmaSaveFSToFile(RSMA_FS(pStat), current); + TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + + // scan and try fix(remove main.db/main.db.xxx and use the one with version) + code = tdRSmaFSScanAndTryFix(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } -/** - * @brief Fetch qtaskfiles LE than version - * - * @param pSma - * @param version - * @param output - * @return int32_t - */ -static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output) { - SVnode *pVnode = pSma->pVnode; - TdDirPtr pDir = NULL; - TdDirEntryPtr pDirEntry = NULL; - char dir[TSDB_FILENAME_LEN]; - const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; - regex_t regex; - int code = 0; - - terrno = TSDB_CODE_SUCCESS; - - tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); - - if (!taosCheckExistFile(dir)) { - smaDebug("vgId:%d, fetch qtask files, no need as dir %s not exist", TD_VID(pVnode), dir); - return TSDB_CODE_SUCCESS; - } - - // Resource allocation and init - if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); - return TSDB_CODE_FAILED; - } - - if (!(pDir = taosOpenDir(dir))) { - regfree(®ex); - terrno = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, fetch qtask files, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); - return TSDB_CODE_FAILED; - } - - int32_t dirLen = strlen(dir); - char *dirEnd = POINTER_SHIFT(dir, dirLen); - regmatch_t regMatch[2]; - while ((pDirEntry = taosReadDir(pDir))) { - char *entryName = taosGetDirEntryName(pDirEntry); - if (!entryName) { - continue; - } +void tdRSmaFSClose(SRSmaFS *pFS) { pFS->aQTaskInf = taosArrayDestroy(pFS->aQTaskInf); } - code = regexec(®ex, entryName, 2, regMatch, 0); +int32_t tdRSmaFSPrepareCommit(SSma *pSma, SRSmaFS *pFSNew) { + int32_t code = 0; + int32_t lino = 0; + char tfname[TSDB_FILENAME_LEN]; - if (code == 0) { - // match - smaInfo("vgId:%d, fetch qtask files, max ver:%" PRIi64 ", %s found", TD_VID(pVnode), version, entryName); + tdRSmaGetCurrentFName(pSma, NULL, tfname); - int64_t ver = -1; - sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &ver); - if ((ver <= version) && (ver > -1)) { - if (!(*output)) { - if (!(*output = taosArrayInit(1, POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } - char *entryDup = strdup(entryName); - if (!entryDup) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - if (!taosArrayPush(*output, &entryDup)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - } else { - } - } else if (code == REG_NOMATCH) { - // not match - smaTrace("vgId:%d, fetch qtask files, not match %s", TD_VID(pVnode), entryName); - continue; - } else { - // has other error - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, fetch qtask files, regexec failed since %s", TD_VID(pVnode), errbuf); - terrno = TSDB_CODE_RSMA_REGEX_MATCH; - goto _end; - } + // generate PRESENT.t + code = tdRSmaSaveFSToFile(pFSNew, tfname); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); } -_end: - taosCloseDir(&pDir); - regfree(®ex); - return terrno == 0 ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; + return code; } -static int32_t tdQTaskFileCmprFn2(const void *p1, const void *p2) { - if (((SQTaskFile *)p1)->version < ((SQTaskFile *)p2)->version) { - return -1; - } else if (((SQTaskFile *)p1)->version > ((SQTaskFile *)p2)->version) { - return 1; +int32_t tdRSmaFSCommit(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + SRSmaFS fs = {0}; + + char current[TSDB_FILENAME_LEN] = {0}; + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, current, current_t); + + if (!taosCheckExistFile(current_t)) { + goto _exit; } - return 0; + // rename the file + if (taosRenameFile(current_t, current) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // load the new FS + code = tdRSmaFSCreate(&fs, 1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaLoadFSFromFile(current, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + + // apply file change + code = tdRSmaFSApplyChange(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + tdRSmaFSClose(&fs); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); + } + return code; } -int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile) { - int32_t code = 0; - int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskFile, tdQTaskFileCmprFn2, TD_GE); +int32_t tdRSmaFSFinishCommit(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pSmaEnv); - if (idx < 0) { - idx = taosArrayGetSize(pFS->aQTaskInf); + taosWLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSCommit(pSma); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(code)); } else { - SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); - int32_t c = tdQTaskFileCmprFn2(pTaskF, qTaskFile); - if (c == 0) { - pTaskF->nRef = qTaskFile->nRef; - pTaskF->version = qTaskFile->version; - pTaskF->size = qTaskFile->size; + smaInfo("vgId:%d, rsmaFS finish commit", SMA_VID(pSma)); + } + return code; +} + +int32_t tdRSmaFSRollback(SSma *pSma) { + int32_t code = 0; + int32_t lino = 0; + + char current_t[TSDB_FILENAME_LEN] = {0}; + tdRSmaGetCurrentFName(pSma, NULL, current_t); + (void)taosRemoveFile(current_t); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pSma), __func__, lino, tstrerror(errno)); + } + return code; +} + +int32_t tdRSmaFSUpsertQTaskFile(SSma *pSma, SRSmaFS *pFS, SQTaskFile *qTaskFile, int32_t nSize) { + int32_t code = 0; + + for (int32_t i = 0; i < nSize; ++i) { + SQTaskFile *qTaskF = qTaskFile + i; + + int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskF, tdQTaskInfCmprFn1, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFS->aQTaskInf); + } else { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); + int32_t c = tdQTaskInfCmprFn1(pTaskF, qTaskF); + if (c == 0) { + if (pTaskF->size != qTaskF->size) { + code = TSDB_CODE_RSMA_FS_UPDATE; + smaError("vgId:%d, %s failed at line %d since %s, level:%" PRIi8 ", suid:%" PRIi64 ", version:%" PRIi64 + ", size:%" PRIi64 " != %" PRIi64, + SMA_VID(pSma), __func__, __LINE__, tstrerror(code), pTaskF->level, pTaskF->suid, pTaskF->version, + pTaskF->size, qTaskF->size); + goto _exit; + } + continue; + } + } + + if (!taosArrayInsert(pFS->aQTaskInf, idx, qTaskF)) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } } - if (taosArrayInsert(pFS->aQTaskInf, idx, qTaskFile) == NULL) { +_exit: + return code; +} + +int32_t tdRSmaFSRef(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + int32_t nRef = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *qFS = RSMA_FS(pStat); + int32_t size = taosArrayGetSize(qFS->aQTaskInf); + + pFS->aQTaskInf = taosArrayInit_s(sizeof(SQTaskFile), size); + if (pFS->aQTaskInf == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *qTaskF = (SQTaskFile *)taosArrayGet(qFS->aQTaskInf, i); + nRef = atomic_fetch_add_32(&qTaskF->nRef, 1); + if (nRef <= 0) { + code = TSDB_CODE_RSMA_FS_REF; + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + memcpy(pFS->aQTaskInf->pData, qFS->aQTaskInf->pData, size * sizeof(SQTaskFile)); + _exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, nRef %d", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code), + nRef); + } return code; -} \ No newline at end of file +} + +void tdRSmaFSUnRef(SSma *pSma, SRSmaFS *pFS) { + int32_t nRef = 0; + char fname[TSDB_FILENAME_LEN]; + SVnode *pVnode = pSma->pVnode; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + + for (int32_t i = 0; i < size; ++i) { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, i); + + nRef = atomic_sub_fetch_32(&pTaskF->nRef, 1); + if (nRef == 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + if (taosRemoveFile(fname) < 0) { + smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), fname, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), fname); + } + } else if (nRef < 0) { + smaWarn("vgId:%d, abnormal unref %s since %s", TD_VID(pVnode), fname, tstrerror(TSDB_CODE_RSMA_FS_REF)); + } + } + + taosArrayDestroy(pFS->aQTaskInf); +} + +int32_t tdRSmaFSTakeSnapshot(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSRef(pSma, pFS); + TSDB_CHECK_CODE(code, lino, _exit); +_exit: + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +int32_t tdRSmaFSCopy(SSma *pSma, SRSmaFS *pFS) { + int32_t code = 0; + int32_t lino = 0; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SRSmaFS *qFS = RSMA_FS(pStat); + int32_t size = taosArrayGetSize(qFS->aQTaskInf); + + code = tdRSmaFSCreate(pFS, size); + TSDB_CHECK_CODE(code, lino, _exit); + taosArrayAddBatch(pFS->aQTaskInf, qFS->aQTaskInf->pData, size); + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pSma->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 2a769b68fe0ff93caa48a6a22030ca02b2af7023..00000cb12914d7a0fbda6b2218c09c1726e51e17 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -34,13 +34,16 @@ static int32_t rsmaRestore(SSma *pSma); SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ if (!RETENTION_VALID(r)) { \ if (l == 0) { \ - goto _err; \ + code = TSDB_CODE_INVALID_PARA; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ break; \ } \ - smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ + TSDB_CHECK_CODE(code, lino, _exit); \ if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback) < 0) { \ - goto _err; \ + code = terrno; \ + TSDB_CHECK_CODE(code, lino, _exit); \ } \ } while (0) @@ -68,12 +71,10 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p days = keepDuration; } - if (level == TSDB_RETENTION_L0) { - goto end; + if (level < TSDB_RETENTION_L1 || level > TSDB_RETENTION_L2) { + goto _exit; } - ASSERT(level >= TSDB_RETENTION_L1 && level <= TSDB_RETENTION_L2); - freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); @@ -91,16 +92,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p if (days < freqDuration) { days = freqDuration; } -end: +_exit: smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); return days; } int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) { + terrno = 0; pKeepCfg->precision = pCfg->precision; switch (type) { case TSDB_TYPE_TSMA: - ASSERT(0); + ASSERTS(0, "undefined smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; case TSDB_TYPE_RSMA_L0: SMA_SET_KEEP_CFG(pVnode, 0); @@ -112,21 +115,22 @@ int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int ty SMA_SET_KEEP_CFG(pVnode, 2); break; default: - ASSERT(0); + ASSERTS(0, "unknown smaType:%d", (int32_t)type); + terrno = TSDB_CODE_APP_ERROR; break; } - return 0; + return terrno; } int32_t smaOpen(SVnode *pVnode, int8_t rollback) { + int32_t code = 0; + int32_t lino = 0; STsdbCfg *pCfg = &pVnode->config.tsdbCfg; - ASSERT(!pVnode->pSma); - SSma *pSma = taosMemoryCalloc(1, sizeof(SSma)); if (!pSma) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } pVnode->pSma = pSma; @@ -137,31 +141,33 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback) { if (VND_IS_RSMA(pVnode)) { STsdbKeepCfg keepCfg = {0}; - for (int i = 0; i < TSDB_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { if (i == TSDB_RETENTION_L0) { SMA_OPEN_RSMA_IMPL(pVnode, 0); } else if (i == TSDB_RETENTION_L1) { SMA_OPEN_RSMA_IMPL(pVnode, 1); } else if (i == TSDB_RETENTION_L2) { SMA_OPEN_RSMA_IMPL(pVnode, 2); - } else { - ASSERT(0); } } // restore the rsma - if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed) < 0) { - goto _err; + if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback) < 0) { + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } } - return 0; -_err: - return -1; +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } int32_t smaClose(SSma *pSma) { if (pSma) { + smaPreClose(pSma); taosThreadMutexDestroy(&pSma->mutex); SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); @@ -181,8 +187,11 @@ int32_t smaClose(SSma *pSma) { * @param committedVer * @return int32_t */ -int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer) { - ASSERT(VND_IS_RSMA(pSma->pVnode)); +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) { + if (!VND_IS_RSMA(pSma->pVnode)) { + terrno = TSDB_CODE_RSMA_INVALID_ENV; + return TSDB_CODE_FAILED; + } - return tdRSmaProcessRestoreImpl(pSma, type, committedVer); + return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback); } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ba99e5515d8b0b15892da433c80305477ab98dd4..37ae7d895e54e5f9b0a4a12c075570d118b4e704 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -21,17 +21,17 @@ #define RSMA_FETCH_ACTIVE_MAX (1000) // ms #define RSMA_FETCH_INTERVAL (5000) // ms +#define RSMA_NEED_FETCH(r) (RSMA_INFO_ITEM((r), 0)->fetchLevel || RSMA_INFO_ITEM((r), 1)->fetchLevel) + SSmaMgmt smaMgmt = { .inited = 0, .rsetId = -1, }; -#define TD_QTASKINFO_FNAME_PREFIX "qinf.v" - typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; -typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static void tdUidStoreDestory(STbUidStore *pStore); static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); @@ -57,38 +57,6 @@ struct SRSmaQTaskInfoItem { void *qTaskInfo; }; -struct SRSmaQTaskInfoIter { - STFile *pTFile; - int64_t offset; - int64_t fsize; - int32_t nBytes; - int32_t nAlloc; - char *pBuf; - // ------------ - char *qBuf; // for iterator - int32_t nBufPos; -}; - -void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t version, char *outputName) { - tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); -} - -void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t version, const char *path, char *outputName) { - tdGetVndFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName); -} - -void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { - tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); -} - -void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { - tdGetVndDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); - int32_t rsmaLen = strlen(outputName); - snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi64 "%s%" PRIi8, suid, TD_DIRSEP, level); -} - static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC if (!taskHandle || !(*taskHandle)) return; @@ -161,12 +129,17 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { } static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { - ASSERT(*pStore == NULL); + if (ASSERTS(*pStore == NULL, "*pStore:%p != NULL", *pStore)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); if (*pStore == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } + return TSDB_CODE_SUCCESS; } @@ -195,15 +168,14 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0)) { + if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", - SMA_VID(pSma), pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); } } @@ -264,8 +236,6 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - ASSERT(ppStore != NULL); - if (!(*ppStore)) { if (tdUidStoreInit(ppStore) < 0) { return TSDB_CODE_FAILED; @@ -292,7 +262,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat // set the backend of stream state tdRSmaQTaskInfoGetFullPathEx(TD_VID(pVnode), pRSmaInfo->suid, idx + 1, tfsGetPrimaryPath(pVnode->pTfs), taskInfDir); if (!taosCheckExistFile(taskInfDir)) { - char *s = strdup(taskInfDir); + char *s = taosStrdup(taskInfDir); if (taosMulMkDir(taosDirName(s)) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosMemoryFree(s); @@ -332,12 +302,15 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - ASSERT(pItem->level > 0); + + if (ASSERTS(pItem->level > 0, "pItem level:%" PRIi8 " should > 0", pItem->level)) { + terrno = TSDB_CODE_APP_ERROR; + return TSDB_CODE_FAILED; + } SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); - pItem->fetchLevel = pItem->level; taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 @@ -363,10 +336,12 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; } +#if 0 if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_TDB_INIT_FAILED; return TSDB_CODE_FAILED; } +#endif SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -374,13 +349,8 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo) { - // TODO: free original pRSmaInfo if exists abnormally - tdFreeRSmaInfo(pSma, *(SRSmaInfo **)pRSmaInfo, true); - if (taosHashRemove(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)) < 0) { - terrno = TSDB_CODE_RSMA_REMOVE_EXISTS; - goto _err; - } - smaWarn("vgId:%d, remove the rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); + smaInfo("vgId:%d, rsma info already exists for table %s, %" PRIi64, SMA_VID(pSma), tbName, suid); + return TSDB_CODE_SUCCESS; } // from write queue: single thead @@ -449,8 +419,8 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { } if (!VND_IS_RSMA(pVnode)) { - smaTrace("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, - pReq->suid); + smaWarn("vgId:%d, not create rsma for stable %s %" PRIi64 " since vnd is not rsma", TD_VID(pVnode), pReq->name, + pReq->suid); return TSDB_CODE_SUCCESS; } @@ -494,9 +464,8 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { tdReleaseRSmaInfo(pSma, pRSmaInfo); - // save to file - // TODO - smaDebug("vgId:%d, drop rsma for table %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); + // no need to save to file as triggered by dropping stable + smaDebug("vgId:%d, drop rsma for stable %" PRIi64 " succeed", TD_VID(pVnode), pReq->suid); return TSDB_CODE_SUCCESS; } @@ -561,7 +530,7 @@ static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) return TSDB_CODE_SUCCESS; } -void tdUidStoreDestory(STbUidStore *pStore) { +static void tdUidStoreDestory(STbUidStore *pStore) { if (pStore) { if (pStore->uidHash) { if (pStore->tbUids) { @@ -601,8 +570,8 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_FAILED; } - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - // TODO: spin lock for race conditiond + SSubmitReq2 *pSubmitReq = (SSubmitReq2 *)pReq; + // spin lock for race condition during insert data if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { return TSDB_CODE_FAILED; } @@ -610,29 +579,19 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { return TSDB_CODE_SUCCESS; } -static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; +static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { + SArray *pSubmitTbData = pMsg ? pMsg->aSubmitTbData : NULL; + int32_t size = taosArrayGetSize(pSubmitTbData); terrno = TSDB_CODE_SUCCESS; - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) { + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pSubmitTbData, i); + if ((terrno = tdUidStorePut(pStore, pData->suid, NULL)) < 0) { return -1; } - - if (!pBlock) break; - tdUidStorePut(pStore, msgIter.suid, NULL); } - if (terrno != TSDB_CODE_SUCCESS) { - return -1; - } return 0; } @@ -708,14 +667,14 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId, - output->info.rows); + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + output->info.id.groupId, output->info.rows); - STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); - SSubmitReq *pReq = NULL; + STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); + SSubmitReq2 *pReq = NULL; // TODO: the schema update should be handled later(TD-17965) - if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { + if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); @@ -723,19 +682,21 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - taosMemoryFreeClear(pReq); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr()); goto _err; } - smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64 - " len %" PRIu32, - SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version, - htonl(pReq->header.contLen)); + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, + SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); - taosMemoryFreeClear(pReq); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } } } @@ -753,22 +714,29 @@ _err: * @brief Copy msg to rsmaQueueBuffer for batch process * * @param pSma + * @param version * @param pMsg + * @param len * @param inputType * @param pInfo * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaImplAsync(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfo *pInfo, - tb_uid_t suid) { - const SSubmitReq *pReq = (const SSubmitReq *)pMsg; +static int32_t tdExecuteRSmaImplAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + SRSmaInfo *pInfo, tb_uid_t suid) { + int32_t size = sizeof(int32_t) + sizeof(int64_t) + len; + void *qItem = taosAllocateQitem(size, DEF_QITEM, 0); - void *qItem = taosAllocateQitem(pReq->header.contLen, DEF_QITEM, 0); if (!qItem) { return TSDB_CODE_FAILED; } - memcpy(qItem, pMsg, pReq->header.contLen); + void *pItem = qItem; + + *(int32_t *)pItem = len; + pItem = POINTER_SHIFT(pItem, sizeof(int32_t)); + *(int64_t *)pItem = version; + memcpy(POINTER_SHIFT(pItem, sizeof(int64_t)), pMsg, len); taosWriteQitem(pInfo->queue, qItem); @@ -840,12 +808,13 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } if (!pInfo->pTSchema) { + terrno = TSDB_CODE_INVALID_PTR; smaWarn("vgId:%d, no schema to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, pInfo->suid); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, - RSMA_INFO_QTASK(pInfo, idx), pInfo->suid); + smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64 " nMsg:%d", SMA_VID(pSma), level, + RSMA_INFO_QTASK(pInfo, idx), pInfo->suid, msgSize); #if 0 for (int32_t i = 0; i < msgSize; ++i) { @@ -854,7 +823,7 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, tdRsmaPrintSubmitReq(pSma, pReq); } #endif - if (qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType) < 0) { + if ((terrno = qSetSMAInput(qTaskInfo, pMsg, msgSize, inputType)) < 0) { smaError("vgId:%d, rsma %" PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } @@ -867,44 +836,51 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, tb_uid_t suid, int8_t idx) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; char *pOutput = NULL; int32_t len = 0; - if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { - smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + if (!srcTaskInfo) { + code = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d, rsma clone, table %" PRIi64 ", no need since srcTaskInfo is NULL", TD_VID(pVnode), suid); + TSDB_CHECK_CODE(code, lino, _exit); } + code = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len); + TSDB_CHECK_CODE(code, lino, _exit); + SReadHandle handle = { .meta = pVnode->pMeta, .vnode = pVnode, .initTqReader = 1, }; - ASSERT(!dstTaskInfo); + + if (ASSERTS(!dstTaskInfo, "dstTaskInfo:%p is not NULL", dstTaskInfo)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!dstTaskInfo) { - terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; + code = TSDB_CODE_RSMA_QTASKINFO_CREATE; + TSDB_CHECK_CODE(code, lino, _exit); } - if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } + code = qDeserializeTaskStatus(dstTaskInfo, pOutput, len); + TSDB_CHECK_CODE(code, lino, _exit); smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); +_exit: taosMemoryFreeClear(pOutput); - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(pOutput); - tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - return TSDB_CODE_FAILED; + if (code) { + tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + } + return code; } /** @@ -915,43 +891,53 @@ _err: * @return int32_t */ static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; SRSmaParam *param = NULL; + SMetaReader mr = {0}; + if (!pInfo) { return TSDB_CODE_SUCCESS; } - SMetaReader mr = {0}; metaReaderInit(&mr, SMA_META(pSma), 0); smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); if (metaGetTableEntryByUidCache(&mr, pInfo->suid) < 0) { - smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == pInfo->suid); + if (mr.me.uid != pInfo->suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (TABLE_IS_ROLLUP(mr.me.flags)) { param = &mr.me.stbEntry.rsmaParam; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (!pInfo->iTaskInfo[i]) { continue; } - if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { - goto _err; - } + code = tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i); + TSDB_CHECK_CODE(code, lino, _exit); } smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); } else { - terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; - goto _err; + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); } +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", flags:%" PRIi8 ",type:%" PRIi8 ", uid:%" PRIi64, + SMA_VID(pSma), __func__, lino, tstrerror(code), pInfo->suid, mr.me.flags, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -_err: - metaReaderClear(&mr); - smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); - return TSDB_CODE_FAILED; + return code; } /** @@ -962,10 +948,14 @@ _err: * @return SRSmaInfo* */ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { + int32_t code = 0; + int32_t lino = 0; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; SRSmaInfo *pRSmaInfo = NULL; + terrno = 0; + if (!pEnv) { terrno = TSDB_CODE_RSMA_INVALID_ENV; return NULL; @@ -985,14 +975,17 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } if (!pRSmaInfo->taskInfo[0]) { - if (tdRSmaInfoClone(pSma, pRSmaInfo) < 0) { + if ((terrno = tdRSmaInfoClone(pSma, pRSmaInfo)) < 0) { taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } } tdRefRSmaInfo(pSma, pRSmaInfo); taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); - ASSERT(pRSmaInfo->suid == suid); + if (ASSERTS(pRSmaInfo->suid == suid, "suid:%" PRIi64 " != %" PRIi64, pRSmaInfo->suid, suid)) { + terrno = TSDB_CODE_APP_ERROR; + return NULL; + } return pRSmaInfo; } taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); @@ -1010,12 +1003,14 @@ static FORCE_INLINE void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { * @brief async mode * * @param pSma + * @param version * @param pMsg * @param inputType * @param suid * @return int32_t */ -static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { +static int32_t tdExecuteRSmaAsync(SSma *pSma, int64_t version, const void *pMsg, int32_t len, int32_t inputType, + tb_uid_t suid) { SRSmaInfo *pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, suid); if (!pRSmaInfo) { smaDebug("vgId:%d, execute rsma, no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -1023,7 +1018,7 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdExecuteRSmaImplAsync(pSma, pMsg, inputType, pRSmaInfo, suid) < 0) { + if (tdExecuteRSmaImplAsync(pSma, version, pMsg, len, inputType, pRSmaInfo, suid) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -1038,40 +1033,43 @@ static int32_t tdExecuteRSmaAsync(SSma *pSma, const void *pMsg, int32_t inputTyp } } } else { - ASSERT(0); + terrno = TSDB_CODE_APP_ERROR; + tdReleaseRSmaInfo(pSma, pRSmaInfo); + smaError("vgId:%d, execute rsma, failed for suid:%" PRIu64 " since %s, type:%d", SMA_VID(pSma), suid, + tstrerror(terrno), inputType); + return TSDB_CODE_FAILED; } tdReleaseRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { +int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, int32_t len, int32_t inputType) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists return TSDB_CODE_SUCCESS; } + STbUidStore uidStore = {0}; - SRetention *pRetention = SMA_RETENTION(pSma); - if (!RETENTION_VALID(pRetention + 1)) { - // return directly if retention level 1 is invalid - return TSDB_CODE_SUCCESS; - } if (inputType == STREAM_INPUT__DATA_SUBMIT) { - if (tdFetchSubmitReqSuids(pMsg, &uidStore) < 0) { + if (tdFetchSubmitReqSuids(pReq, &uidStore) < 0) { + smaError("vgId:%d, failed to process rsma submit fetch suid since: %s", SMA_VID(pSma), terrstr()); goto _err; } if (uidStore.suid != 0) { - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, uidStore.suid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, uidStore.suid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 1 since: %s", SMA_VID(pSma), terrstr()); goto _err; } void *pIter = NULL; while ((pIter = taosHashIterate(uidStore.uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - if (tdExecuteRSmaAsync(pSma, pMsg, inputType, *pTbSuid) < 0) { + if (tdExecuteRSmaAsync(pSma, version, pMsg, len, inputType, *pTbSuid) < 0) { + smaError("vgId:%d, failed to process rsma submit exec 2 since: %s", SMA_VID(pSma), terrstr()); goto _err; } } @@ -1081,7 +1079,6 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - smaError("vgId:%d, failed to process rsma submit since: %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } @@ -1093,19 +1090,22 @@ _err: * @return int32_t */ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SArray *suidList = NULL; STbUidStore uidStore = {0}; SMetaReader mr = {0}; + tb_uid_t suid = 0; if (!(suidList = taosArrayInit(1, sizeof(tb_uid_t)))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } if (vnodeGetStbIdList(pSma->pVnode, 0, suidList) < 0) { - smaError("vgId:%d, failed to restore rsma env since get stb id list error: %s", TD_VID(pVnode), terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } int64_t arrSize = taosArrayGetSize(suidList); @@ -1122,19 +1122,26 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { int64_t nRsmaTables = 0; metaReaderInit(&mr, SMA_META(pSma), 0); if (!(uidStore.tbUids = taosArrayInit(1024, sizeof(tb_uid_t)))) { - goto _err; + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } + for (int64_t i = 0; i < arrSize; ++i) { - tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + suid = *(tb_uid_t *)taosArrayGet(suidList, i); smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid); if (metaGetTableEntryByUidCache(&mr, suid) < 0) { - smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } tDecoderClear(&mr.coder); - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == suid); + if (mr.me.type != TSDB_SUPER_TABLE) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } + if (mr.me.uid != suid) { + code = TSDB_CODE_RSMA_INVALID_SCHEMA; + TSDB_CHECK_CODE(code, lino, _exit); + } if (TABLE_IS_ROLLUP(mr.me.flags)) { ++nRsmaTables; SRSmaParam *param = &mr.me.stbEntry.rsmaParam; @@ -1144,22 +1151,20 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); } if (tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { - smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // reload all ctbUids for suid uidStore.suid = suid; if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) { - smaError("vgId:%d, rsma restore, get ctb idlist failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } if (tdUpdateTbUidList(pVnode->pSma, &uidStore, true) < 0) { - smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayClear(uidStore.tbUids); @@ -1168,36 +1173,36 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } } - metaReaderClear(&mr); - taosArrayDestroy(suidList); - tdUidStoreDestory(&uidStore); - if (nTables) { *nTables = nRsmaTables; } - - return TSDB_CODE_SUCCESS; -_err: +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, suid:%" PRIi64 ", type:%" PRIi8 ", uid:%" PRIi64, TD_VID(pVnode), + __func__, lino, tstrerror(code), suid, mr.me.type, mr.me.uid); + } metaReaderClear(&mr); taosArrayDestroy(suidList); tdUidStoreDestory(&uidStore); - - return TSDB_CODE_FAILED; + return code; } /** - * @brief reload ts data from checkpoint - * - * @param pSma - * @return int32_t + * N.B. the data would be restored from the unified WAL replay procedure */ -static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) { - // NOTHING TODO: the data would be restored from the unified WAL replay procedure - return TSDB_CODE_SUCCESS; -} +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback) { + // step 1: init env + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } -int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) { - // step 1: iterate all stables to restore the rsma env + // step 2: open SRSmaFS for qTaskFiles + if (tdRSmaFSOpen(pSma, qtaskFileVer, rollback) < 0) { + goto _err; + } + + // step 3: iterate all stables to restore the rsma env int64_t nTables = 0; if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { goto _err; @@ -1207,16 +1212,6 @@ int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) return TSDB_CODE_SUCCESS; } - // step 2: reload ts data from checkpoint - if (tdRSmaRestoreTSDataReload(pSma) < 0) { - goto _err; - } - - // step 3: open SRSmaFS for qTaskFiles - if (tdRSmaFSOpen(pSma, qtaskFileVer) < 0) { - goto _err; - } - smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed", SMA_VID(pSma), type, qtaskFileVer); return TSDB_CODE_SUCCESS; _err: @@ -1226,19 +1221,26 @@ _err: } int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { - SSma *pSma = pRSmaStat->pSma; - SVnode *pVnode = pSma->pVnode; - int32_t vid = SMA_VID(pSma); + int32_t code = 0; + int32_t lino = 0; + SSma *pSma = pRSmaStat->pSma; + SVnode *pVnode = pSma->pVnode; + SArray *qTaskFArray = NULL; + int64_t version = pRSmaStat->commitAppliedVer; + TdFilePtr pOutFD = NULL; + TdFilePtr pInFD = NULL; + char fname[TSDB_FILENAME_LEN]; + char fnameVer[TSDB_FILENAME_LEN]; + SRSmaFS fs = {0}; if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); - if (pRSmaStat->commitAppliedVer <= fsMaxVer) { - smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, - pRSmaStat->commitAppliedVer, fsMaxVer); - return TSDB_CODE_SUCCESS; + qTaskFArray = taosArrayInit(taosHashGetSize(pInfoHash) << 1, sizeof(SQTaskFile)); + if (!qTaskFArray) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } void *infoHash = NULL; @@ -1253,19 +1255,79 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); if (pItem && pItem->pStreamState) { if (streamStateCommit(pItem->pStreamState) < 0) { - terrno = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; - goto _err; + code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; + TSDB_CHECK_CODE(code, lino, _exit); + } + smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode), + pRSmaInfo->suid, i + 1); + + // qTaskInfo file + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, -1, tfsGetPrimaryPath(pVnode->pTfs), fname); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pRSmaInfo->suid, i + 1, version, tfsGetPrimaryPath(pVnode->pTfs), + fnameVer); + if (taosCheckExistFile(fnameVer)) { + smaWarn("vgId:%d, rsma persist, duplicate file %s exist", TD_VID(pVnode), fnameVer); + } + + pOutFD = taosCreateFile(fnameVer, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); } - smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 " level %d", vid, pRSmaInfo->suid, - i + 1); + pInFD = taosOpenFile(fname, TD_FILE_READ); + if (pInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size = 0; + uint32_t mtime = 0; + if (taosFStatFile(pInFD, &size, &mtime) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t offset = 0; + if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, rsma persist, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), fname, + fnameVer, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&pOutFD); + taosCloseFile(&pInFD); + + SQTaskFile qTaskF = { + .nRef = 1, .level = i + 1, .suid = pRSmaInfo->suid, .version = version, .size = size, .mtime = mtime}; + + taosArrayPush(qTaskFArray, &qTaskF); } } } - return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); - return TSDB_CODE_FAILED; + // prepare + code = tdRSmaFSCopy(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaFSUpsertQTaskFile(pSma, &fs, qTaskFArray->pData, taosArrayGetSize(qTaskFArray)); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaFSPrepareCommit(pSma, &fs); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + + taosArrayDestroy(fs.aQTaskInf); + taosArrayDestroy(qTaskFArray); + + if (code) { + if (pOutFD) taosCloseFile(&pOutFD); + if (pInFD) taosCloseFile(&pInFD); + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + + terrno = code; + return code; } /** @@ -1343,21 +1405,18 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { // async process pItem->fetchLevel = pItem->level; #if 0 + // debugging codes SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); - ASSERT(qItem->level == pItem->level); - ASSERT(qItem->fetchLevel == pItem->fetchLevel); + make sure(qItem->level == pItem->level); + make sure(qItem->fetchLevel == pItem->fetchLevel); #endif if (atomic_load_8(&pRSmaInfo->assigned) == 0) { tsem_post(&(pStat->notEmpty)); } } break; - case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is paused", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); - } break; case TASK_TRIGGER_STAT_INACTIVE: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is inactive", + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is inactive ", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; case TASK_TRIGGER_STAT_INIT: { @@ -1365,8 +1424,9 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SMA_VID(pSma), pItem->level, pRSmaInfo->suid); } break; default: { - smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat is unknown", - SMA_VID(pSma), pItem->level, pRSmaInfo->suid); + smaDebug("vgId:%d, rsma fetch task not start for level:%" PRIi8 " suid:%" PRIi64 " since stat:%" PRIi8 + " is unknown", + SMA_VID(pSma), pItem->level, pRSmaInfo->suid, fetchTriggerStat); } break; } @@ -1378,7 +1438,8 @@ _end: static void tdFreeRSmaSubmitItems(SArray *pItems) { for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { - taosFreeQitem(*(void **)taosArrayGet(pItems, i)); + SPackedData *packData = taosArrayGet(pItems, i); + taosFreeQitem(POINTER_SHIFT(packData->msgStr, -sizeof(int32_t) - sizeof(int64_t))); } taosArrayClear(pItems); } @@ -1447,7 +1508,11 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA void *msg = NULL; taosGetQitem(qall, (void **)&msg); if (msg) { - if (!taosArrayPush(pSubmitArr, &msg)) { + SPackedData packData = {.msgLen = *(int32_t *)msg, + .ver = *(int64_t *)POINTER_SHIFT(msg, sizeof(int32_t)), + .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; + + if (!taosArrayPush(pSubmitArr, &packData)) { tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } @@ -1460,7 +1525,6 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA if (size > 0) { for (int32_t i = 1; i <= TSDB_RETENTION_L2; ++i) { if (tdExecuteRSmaImpl(pSma, pSubmitArr->pData, size, STREAM_INPUT__MERGED_SUBMIT, pInfo, type, i) < 0) { - tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } } @@ -1468,6 +1532,9 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA } return TSDB_CODE_SUCCESS; _err: + smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid, + type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr()); + tdFreeRSmaSubmitItems(pSubmitArr); while (1) { void *msg = NULL; taosGetQitem(qall, (void **)&msg); @@ -1489,6 +1556,8 @@ _err: */ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { + int32_t code = 0; + int32_t lino = 0; SVnode *pVnode = pSma->pVnode; SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); @@ -1497,14 +1566,14 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { bool isFetchAll = false; if (!pRSmaStat || !(infoHash = RSMA_INFO_HASH(pRSmaStat))) { - terrno = TSDB_CODE_RSMA_INVALID_STAT; - goto _err; + code = TSDB_CODE_RSMA_INVALID_STAT; + TSDB_CHECK_CODE(code, lino, _exit); } if (!(pSubmitArr = - taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), POINTER_BYTES))) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + taosArrayInit(TMIN(RSMA_SUBMIT_BATCH_SIZE, atomic_load_64(&pRSmaStat->nBufItems)), sizeof(SPackedData)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); } while (true) { @@ -1514,8 +1583,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { while ((pIter = taosHashIterate(infoHash, pIter))) { SRSmaInfo *pInfo = *(SRSmaInfo **)pIter; if (atomic_val_compare_exchange_8(&pInfo->assigned, 0, 1) == 0) { - if ((taosQueueItemSize(pInfo->queue) > 0) || RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || - RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + if ((taosQueueItemSize(pInfo->queue) > 0) || RSMA_NEED_FETCH(pInfo)) { int32_t batchCnt = -1; int32_t batchMax = taosHashGetSize(infoHash) / tsNumOfVnodeRsmaThreads; bool occupied = (batchMax <= 1); @@ -1531,13 +1599,24 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { smaDebug("vgId:%d, batchSize:%d, execType:%" PRIi32, SMA_VID(pSma), qallItemSize, type); } - if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + if (RSMA_NEED_FETCH(pInfo)) { int8_t oldStat = atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 2); if (oldStat == 0 || ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { int32_t oldVal = atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); - ASSERT(oldVal >= 0); - tdRSmaFetchAllResult(pSma, pInfo); + + if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) { + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)); + if (curStat == 1) { + smaDebug("vgId:%d, fetch all not exec as commit stat is %" PRIi8, SMA_VID(pSma), curStat); + } else { + tdRSmaFetchAllResult(pSma, pInfo); + } + if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); } @@ -1547,17 +1626,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if (qallItemSize > 0) { atomic_fetch_sub_64(&pRSmaStat->nBufItems, qallItemSize); continue; - } else if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { - if (atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)) == 0) { - continue; - } - for (int32_t j = 0; j < TSDB_RETENTION_L2; ++j) { - SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, j); - if (pItem->fetchLevel) { - pItem->fetchLevel = 0; - taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - } - } + } + if (RSMA_NEED_FETCH(pInfo)) { + continue; } break; @@ -1567,7 +1638,9 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } } } else { - ASSERT(0); + ASSERTS(0, "unknown rsma exec type:%d", (int32_t)type); + code = TSDB_CODE_APP_ERROR; + TSDB_CHECK_CODE(code, lino, _exit); } if (atomic_load_64(&pRSmaStat->nBufItems) <= 0) { @@ -1586,10 +1659,10 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { } // end of while(true) -_end: +_exit: taosArrayDestroy(pSubmitArr); - return TSDB_CODE_SUCCESS; -_err: - taosArrayDestroy(pSubmitArr); - return TSDB_CODE_FAILED; + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 0a6fac0fe7c2d6c724855907e61f498a82abbd57..c00e96a06664db0a60184fdb09e16ee0b68c3d45 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -17,14 +17,13 @@ static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppData); static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version); -static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader); // SRSmaSnapReader ======================================== struct SRSmaSnapReader { SSma* pSma; int64_t sver; int64_t ever; + SRSmaFS fs; // for data file int8_t rsmaDataDone[TSDB_RETENTION_L2]; @@ -32,19 +31,23 @@ struct SRSmaSnapReader { // for qtaskinfo file int8_t qTaskDone; + int32_t fsIter; SQTaskFReader* pQTaskFReader; }; int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader) { int32_t code = 0; + int32_t lino = 0; SVnode* pVnode = pSma->pVnode; SRSmaSnapReader* pReader = NULL; + SSmaEnv* pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat* pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); // alloc pReader = (SRSmaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pReader->pSma = pSma; pReader->sver = sver; @@ -55,171 +58,147 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapRead if (pSma->pRSmaTsdb[i]) { code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2, &pReader->pDataReader[i]); - if (code < 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } } // open qtaskinfo - if ((code = rsmaQTaskInfSnapReaderOpen(pReader, ever)) < 0) { - goto _err; + taosRLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSRef(pSma, &pReader->fs); + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(pReader->fs.aQTaskInf) > 0) { + pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); + if (!pReader->pQTaskFReader) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + pReader->pQTaskFReader->pSma = pSma; + pReader->pQTaskFReader->version = pReader->ever; } *ppReader = pReader; - - return TSDB_CODE_SUCCESS; -_err: - if (pReader) rsmaSnapReaderClose(&pReader); - *ppReader = NULL; - smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code)); - return TSDB_CODE_FAILED; -} - -static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version) { - int32_t code = 0; - SSma* pSma = pReader->pSma; - SVnode* pVnode = pSma->pVnode; - SSmaEnv* pEnv = NULL; - SRSmaStat* pStat = NULL; - - if (!(pEnv = SMA_RSMA_ENV(pSma))) { - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as env is NULL", - TD_VID(pVnode), version); - return TSDB_CODE_SUCCESS; - } - - pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); - - int32_t ref = tdRSmaFSRef(pReader->pSma, pStat, version); - if (ref < 1) { - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as ref is %d", - TD_VID(pVnode), version, ref); - return TSDB_CODE_SUCCESS; - } - - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - - if (!taosCheckExistFile(qTaskInfoFullName)) { - tdRSmaFSUnRef(pSma, pStat, version); - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exist", - TD_VID(pVnode), version, qTaskInfoFullName); - return TSDB_CODE_SUCCESS; - } - - pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); - if (!pReader->pQTaskFReader) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - - TdFilePtr fp = taosOpenFile(qTaskInfoFullName, TD_FILE_READ); - if (!fp) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFreeClear(pReader->pQTaskFReader); - goto _end; - } - - pReader->pQTaskFReader->pReadH = fp; - pReader->pQTaskFReader->pSma = pSma; - pReader->pQTaskFReader->version = pReader->ever; - -_end: - if (code < 0) { - tdRSmaFSUnRef(pSma, pStat, version); - smaError("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); - return TSDB_CODE_FAILED; - } - - smaInfo("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); - return TSDB_CODE_SUCCESS; -} - -static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader) { - if (!(*ppReader)) { - return TSDB_CODE_SUCCESS; +_exit: + if (code) { + if (pReader) rsmaSnapReaderClose(&pReader); + *ppReader = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); } - - SSma* pSma = (*ppReader)->pSma; - SRSmaStat* pStat = SMA_RSMA_STAT(pSma); - int64_t version = (*ppReader)->version; - - taosCloseFile(&(*ppReader)->pReadH); - tdRSmaFSUnRef(pSma, pStat, version); - taosMemoryFreeClear(*ppReader); - smaInfo("vgId:%d, vnode snapshot rsma reader closed for qTaskInfo version %" PRIi64, SMA_VID(pSma), version); - - return TSDB_CODE_SUCCESS; + return code; } static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppBuf) { int32_t code = 0; - SSma* pSma = pReader->pSma; + int32_t lino = 0; + SVnode* pVnode = pReader->pSma->pVnode; + SQTaskFReader* qReader = pReader->pQTaskFReader; + SRSmaFS* pFS = &pReader->fs; int64_t n = 0; uint8_t* pBuf = NULL; - SQTaskFReader* qReader = pReader->pQTaskFReader; + int64_t version = pReader->ever; + char fname[TSDB_FILENAME_LEN]; if (!qReader) { *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, qTaskReader is NULL", SMA_VID(pSma)); - return 0; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since qTaskReader is NULL", TD_VID(pVnode)); + goto _exit; + } + + if (pReader->fsIter >= taosArrayGetSize(pFS->aQTaskInf)) { + *ppBuf = NULL; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, fsIter reach end", TD_VID(pVnode)); + goto _exit; + } + + while (pReader->fsIter < taosArrayGetSize(pFS->aQTaskInf)) { + SQTaskFile* qTaskF = taosArrayGet(pFS->aQTaskInf, pReader->fsIter++); + if (qTaskF->version != version) { + continue; + } + + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, tfsGetPrimaryPath(pVnode->pTfs), + fname); + if (!taosCheckExistFile(fname)) { + smaError("vgId:%d, vnode snapshot rsma reader for qtaskinfo, table %" PRIi64 ", level %" PRIi8 + ", version %" PRIi64 " failed since %s not exist", + TD_VID(pVnode), qTaskF->suid, qTaskF->level, version, fname); + code = TSDB_CODE_RSMA_FS_SYNC; + TSDB_CHECK_CODE(code, lino, _exit); + } + + TdFilePtr fp = taosOpenFile(fname, TD_FILE_READ); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + qReader->pReadH = fp; + qReader->level = qTaskF->level; + qReader->suid = qTaskF->suid; } if (!qReader->pReadH) { *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, readh is NULL", SMA_VID(pSma)); - return 0; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, not needed since readh is NULL", TD_VID(pVnode)); + goto _exit; } int64_t size = 0; if (taosFStatFile(qReader->pReadH, &size, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // seek if (taosLSeekFile(qReader->pReadH, 0, SEEK_SET) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - ASSERT(!(*ppBuf)); - // alloc - *ppBuf = taosMemoryCalloc(1, sizeof(SSnapDataHdr) + size); + if (*ppBuf) { + *ppBuf = taosMemoryRealloc(*ppBuf, sizeof(SSnapDataHdr) + size); + } else { + *ppBuf = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + } if (!(*ppBuf)) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // read n = taosReadFile(qReader->pReadH, POINTER_SHIFT(*ppBuf, sizeof(SSnapDataHdr)), size); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else if (n != size) { code = TSDB_CODE_FILE_CORRUPTED; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, size:%" PRIi64, SMA_VID(pSma), size); + smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo, version:%" PRIi64 ", size:%" PRIi64, TD_VID(pVnode), version, + size); SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppBuf); pHdr->type = SNAP_DATA_QTASK; + pHdr->flag = qReader->level; + pHdr->index = qReader->suid; pHdr->size = size; _exit: - smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", SMA_VID(pSma)); - return code; + if (qReader) taosCloseFile(&qReader->pReadH); -_err: - *ppBuf = NULL; - smaError("vgId:%d, vnode snapshot rsma read qtaskinfo failed since %s", SMA_VID(pSma), tstrerror(code)); + if (code) { + *ppBuf = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + smaInfo("vgId:%d, vnode snapshot rsma read qtaskinfo succeed", TD_VID(pVnode)); + } return code; } int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + int32_t lino = 0; *ppData = NULL; @@ -233,14 +212,11 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { if (!pReader->rsmaDataDone[i]) { smaInfo("vgId:%d, vnode snapshot rsma read level %d not done", SMA_VID(pReader->pSma), i); code = tsdbSnapRead(pTsdbSnapReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { - if (*ppData) { - goto _exit; - } else { - pReader->rsmaDataDone[i] = 1; - } + pReader->rsmaDataDone[i] = 1; } } else { smaInfo("vgId:%d, vnode snapshot rsma read level %d is done", SMA_VID(pReader->pSma), i); @@ -251,22 +227,21 @@ int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { if (!pReader->qTaskDone) { smaInfo("vgId:%d, vnode snapshot rsma qtaskinfo not done", SMA_VID(pReader->pSma)); code = rsmaSnapReadQTaskInfo(pReader, ppData); - if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); + if (*ppData) { + goto _exit; } else { pReader->qTaskDone = 1; - if (*ppData) { - goto _exit; - } } } _exit: - smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); - return code; - -_err: - smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + if (code) { + smaError("vgId:%d, vnode snapshot rsma read failed since %s", SMA_VID(pReader->pSma), tstrerror(code)); + rsmaSnapReaderClose(&pReader); + } else { + smaInfo("vgId:%d, vnode snapshot rsma read succeed", SMA_VID(pReader->pSma)); + } return code; } @@ -274,14 +249,15 @@ int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { int32_t code = 0; SRSmaSnapReader* pReader = *ppReader; + tdRSmaFSUnRef(pReader->pSma, &pReader->fs); + taosMemoryFreeClear(pReader->pQTaskFReader); + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pReader->pDataReader[i]) { tsdbSnapReaderClose(&pReader->pDataReader[i]); } } - rsmaQTaskInfSnapReaderClose(&pReader->pQTaskFReader); - smaInfo("vgId:%d, vnode snapshot rsma reader closed", SMA_VID(pReader->pSma)); taosMemoryFreeClear(*ppReader); @@ -293,28 +269,23 @@ struct SRSmaSnapWriter { SSma* pSma; int64_t sver; int64_t ever; - - // config - int64_t commitID; + SRSmaFS fs; // for data file STsdbSnapWriter* pDataWriter[TSDB_RETENTION_L2]; - - // for qtaskinfo file - SQTaskFReader* pQTaskFReader; - SQTaskFWriter* pQTaskFWriter; }; int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter) { int32_t code = 0; - SRSmaSnapWriter* pWriter = NULL; + int32_t lino = 0; SVnode* pVnode = pSma->pVnode; + SRSmaSnapWriter* pWriter = NULL; // alloc pWriter = (SRSmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { + if (!pWriter) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pWriter->pSma = pSma; pWriter->sver = sver; @@ -324,100 +295,152 @@ int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWrit for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapWriterOpen(pSma->pRSmaTsdb[i], sver, ever, &pWriter->pDataWriter[i]); - if (code < 0) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); } } // qtaskinfo - SQTaskFWriter* qWriter = (SQTaskFWriter*)taosMemoryCalloc(1, sizeof(SQTaskFWriter)); - qWriter->pSma = pSma; - - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), 0, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - TdFilePtr qTaskF = taosCreateFile(qTaskInfoFullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!qTaskF) { - taosMemoryFree(qWriter); - code = TAOS_SYSTEM_ERROR(errno); - smaError("vgId:%d, rsma snapshot writer open %s failed since %s", TD_VID(pSma->pVnode), qTaskInfoFullName, - tstrerror(code)); - goto _err; - } - qWriter->pWriteH = qTaskF; - int32_t fnameLen = strlen(qTaskInfoFullName) + 1; - qWriter->fname = taosMemoryCalloc(1, fnameLen); - strncpy(qWriter->fname, qTaskInfoFullName, fnameLen); - pWriter->pQTaskFWriter = qWriter; - smaDebug("vgId:%d, rsma snapshot writer open succeed for %s", TD_VID(pSma->pVnode), qTaskInfoFullName); + code = tdRSmaFSCopy(pSma, &pWriter->fs); + TSDB_CHECK_CODE(code, lino, _exit); // snapWriter *ppWriter = pWriter; - - smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); +_exit: + if (code) { + if (pWriter) rsmaSnapWriterClose(&pWriter, 0); + *ppWriter = NULL; + smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + smaInfo("vgId:%d, rsma snapshot writer open succeed", TD_VID(pSma->pVnode)); + } return code; +} -_err: - smaError("vgId:%d, rsma snapshot writer open failed since %s", TD_VID(pSma->pVnode), tstrerror(code)); - if (pWriter) rsmaSnapWriterClose(&pWriter, 0); - *ppWriter = NULL; +int32_t rsmaSnapWriterPrepareClose(SRSmaSnapWriter* pWriter) { + int32_t code = 0; + int32_t lino = 0; + + if (pWriter) { + code = tdRSmaFSPrepareCommit(pWriter->pSma, &pWriter->fs); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + smaError("vgId:%d, %s failed at line %d since %s", SMA_VID(pWriter->pSma), __func__, lino, tstrerror(code)); + } return code; } int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; + int32_t lino = 0; + SSma* pSma = NULL; + SVnode* pVnode = NULL; + SSmaEnv* pEnv = NULL; + SRSmaStat* pStat = NULL; SRSmaSnapWriter* pWriter = *ppWriter; - SVnode* pVnode = pWriter->pSma->pVnode; + const char* primaryPath = NULL; + char fname[TSDB_FILENAME_LEN] = {0}; + char fnameVer[TSDB_FILENAME_LEN] = {0}; + TdFilePtr pOutFD = NULL; + TdFilePtr pInFD = NULL; + + if (!pWriter) { + goto _exit; + } - if (rollback) { - // TODO: rsma1/rsma2 - // qtaskinfo - if (pWriter->pQTaskFWriter) { - if (taosRemoveFile(pWriter->pQTaskFWriter->fname) != 0) { - smaWarn("vgId:%d, vnode snapshot rsma writer failed to remove %s since %s", SMA_VID(pWriter->pSma), - pWriter->pQTaskFWriter->fname ? pWriter->pQTaskFWriter->fname : "NULL", - tstrerror(TAOS_SYSTEM_ERROR(errno))); - } + pSma = pWriter->pSma; + pVnode = pSma->pVnode; + pEnv = SMA_RSMA_ENV(pSma); + pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); + primaryPath = tfsGetPrimaryPath(pVnode->pTfs); + + // rsma1/rsma2 + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pWriter->pDataWriter[i]) { + code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); + TSDB_CHECK_CODE(code, lino, _exit); } + } + + // qtaskinfo + if (rollback) { + tdRSmaFSRollback(pSma); + // remove qTaskFiles } else { - // rsma1/rsma2 - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (pWriter->pDataWriter[i]) { - code = tsdbSnapWriterClose(&pWriter->pDataWriter[i], rollback); - if (code) goto _err; + // sendFile from fname.Ver to fname + SRSmaFS* pFS = &pWriter->fs; + int32_t size = taosArrayGetSize(pFS->aQTaskInf); + for (int32_t i = 0; i < size; ++i) { + SQTaskFile* pTaskF = TARRAY_GET_ELEM(pFS->aQTaskInf, i); + if (pTaskF->version == pWriter->ever) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, pTaskF->version, primaryPath, fnameVer); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->suid, pTaskF->level, -1, primaryPath, fname); + + pInFD = taosOpenFile(fnameVer, TD_FILE_READ); + if (pInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + pOutFD = taosCreateFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t size = 0; + if (taosFStatFile(pInFD, &size, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t offset = 0; + if (taosFSendFile(pOutFD, pInFD, &offset, size) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, vnode snapshot rsma writer, send qtaskinfo file %s to %s failed since %s", TD_VID(pVnode), + fnameVer, fname, tstrerror(code)); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosCloseFile(&pOutFD); + taosCloseFile(&pInFD); } } - // qtaskinfo - if (pWriter->pQTaskFWriter) { - char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pWriter->ever, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); - if (taosRenameFile(pWriter->pQTaskFWriter->fname, qTaskInfoFullName) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma writer rename %s to %s", SMA_VID(pWriter->pSma), - pWriter->pQTaskFWriter->fname, qTaskInfoFullName); - // rsma restore - if ((code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever)) < 0) { - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma writer restore from %s succeed", SMA_VID(pWriter->pSma), qTaskInfoFullName); + // lock + taosWLockLatch(RSMA_FS_LOCK(pStat)); + code = tdRSmaFSCommit(pSma); + if (code) { + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + goto _exit; } + // unlock + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); } - smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", SMA_VID(pWriter->pSma)); - taosMemoryFree(pWriter); + // rsma restore + code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever, rollback); + TSDB_CHECK_CODE(code, lino, _exit); + smaInfo("vgId:%d, vnode snapshot rsma writer restore from sync succeed", SMA_VID(pSma)); + +_exit: + if (pWriter) taosMemoryFree(pWriter); *ppWriter = NULL; - return code; + if (code) { + if (pOutFD) taosCloseFile(&pOutFD); + if (pInFD) taosCloseFile(&pInFD); + smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pSma), tstrerror(code)); + } else { + smaInfo("vgId:%d, vnode snapshot rsma writer close succeed", pSma ? SMA_VID(pSma) : 0); + } -_err: - smaError("vgId:%d, vnode snapshot rsma writer close failed since %s", SMA_VID(pWriter->pSma), tstrerror(code)); return code; } int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; + int32_t lino = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // rsma1/rsma2 @@ -430,42 +453,81 @@ int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) } else if (pHdr->type == SNAP_DATA_QTASK) { code = rsmaSnapWriteQTaskInfo(pWriter, pData, nData); } else { - ASSERT(0); + code = TSDB_CODE_RSMA_FS_SYNC; } - if (code < 0) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - smaInfo("vgId:%d, rsma snapshot write for data type %" PRIi8 " succeed", SMA_VID(pWriter->pSma), pHdr->type); - return code; - -_err: - smaError("vgId:%d, rsma snapshot write for data type %" PRIi8 " failed since %s", SMA_VID(pWriter->pSma), pHdr->type, - tstrerror(code)); + if (code) { + smaError("vgId:%d, %s failed at line %d since %s, data type %" PRIi8, SMA_VID(pWriter->pSma), __func__, lino, + tstrerror(code), pHdr->type); + } else { + smaInfo("vgId:%d, rsma snapshot write for data type %" PRIi8 " succeed", SMA_VID(pWriter->pSma), pHdr->type); + } return code; } static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - SQTaskFWriter* qWriter = pWriter->pQTaskFWriter; - - if (qWriter && qWriter->pWriteH) { - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - int64_t size = pHdr->size; - ASSERT(size == (nData - sizeof(SSnapDataHdr))); - int64_t contLen = taosWriteFile(qWriter->pWriteH, pHdr->data, size); - if (contLen != size) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", SMA_VID(pWriter->pSma), qWriter->fname); + int32_t code = 0; + int32_t lino = 0; + SSma* pSma = pWriter->pSma; + SVnode* pVnode = pSma->pVnode; + char fname[TSDB_FILENAME_LEN]; + TdFilePtr fp = NULL; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + + fname[0] = '\0'; + + if (pHdr->size != (nData - sizeof(SSnapDataHdr))) { + code = TSDB_CODE_RSMA_FS_SYNC; + TSDB_CHECK_CODE(code, lino, _exit); + } + + SQTaskFile qTaskFile = { + .nRef = 1, .level = pHdr->flag, .suid = pHdr->index, .version = pWriter->ever, .size = pHdr->size}; + + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pHdr->index, pHdr->flag, qTaskFile.version, + tfsGetPrimaryPath(pVnode->pTfs), fname); + + fp = taosCreateFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t contLen = taosWriteFile(fp, pHdr->data, pHdr->size); + if (contLen != pHdr->size) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + uint32_t mtime = 0; + if (taosFStatFile(fp, NULL, &mtime) != 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); } else { - smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo is not needed", SMA_VID(pWriter->pSma)); + qTaskFile.mtime = mtime; } + if (taosFsyncFile(fp) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosCloseFile(&fp); + + code = tdRSmaFSUpsertQTaskFile(pSma, &pWriter->fs, &qTaskFile, 1); + TSDB_CHECK_CODE(code, lino, _exit); + _exit: - return code; + if (code) { + if (fp) { + (void)taosRemoveFile(fname); + } + smaError("vgId:%d, %s failed at line %d since %s, file:%s", TD_VID(pVnode), __func__, lino, tstrerror(code), fname); + } else { + smaInfo("vgId:%d, vnode snapshot rsma write qtaskinfo %s succeed", TD_VID(pVnode), fname); + } -_err: - smaError("vgId:%d, vnode snapshot rsma write qtaskinfo failed since %s", SMA_VID(pWriter->pSma), tstrerror(code)); return code; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index a2c9484693a720243ccb205f0d311289c86f254e..5058a7fc76e16c85881a2f7fad30ca8109888263 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -14,6 +14,7 @@ */ #include "sma.h" +#include "tq.h" #include "tsdb.h" #define SMA_STORAGE_MINUTES_MAX 86400 @@ -24,14 +25,13 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char * static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); -// TODO: Who is responsible for resource allocate and release? int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); } - // TODO: destroy SSDataBlocks(msg) + return code; } @@ -41,7 +41,6 @@ int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); } - // TODO: destroy SSDataBlocks(msg) return code; } @@ -111,35 +110,240 @@ _err: * @return int32_t */ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { - SSmaCfg *pCfg = (SSmaCfg *)pMsg; + int32_t code = 0; + int32_t lino = 0; + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + SName stbFullName = {0}; + SVCreateStbReq pReq = {0}; if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { // create tsma meta in dstVgId if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // create stable to save tsma result in dstVgId - SName stbFullName = {0}; tNameFromString(&stbFullName, pCfg->dstTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVCreateStbReq pReq = {0}; pReq.name = (char *)tNameGetTableName(&stbFullName); pReq.suid = pCfg->dstTbUid; pReq.schemaRow = pCfg->schemaRow; pReq.schemaTag = pCfg->schemaTag; if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } + } else { + code = terrno = TSDB_CODE_TSMA_INVALID_STAT; + TSDB_CHECK_CODE(code, lino, _exit); + } +_exit: + if (code) { + smaError("vgId:%d, failed at line %d to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 + " dstTb:%s dstVg:%d", + SMA_VID(pSma), lino, pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, + pCfg->dstVgId); + } else { smaDebug("vgId:%d, success to create sma index %s %" PRIi64 " on stb:%" PRIi64 ", dstSuid:%" PRIi64 " dstTb:%s dstVg:%d", SMA_VID(pSma), pCfg->indexName, pCfg->indexUid, pCfg->tableUid, pCfg->dstTbUid, pReq.name, pCfg->dstVgId); - } else { - ASSERT(0); } - return 0; + return code; +} + +int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, + SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, + SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) { + void *pBuf = NULL; + int32_t len = 0; + SSubmitReq2 *pReq = NULL; + SArray *tagArray = NULL; + SArray *createTbArray = NULL; + SArray *pVals = NULL; + + int32_t sz = taosArrayGetSize(pBlocks); + + if (!(tagArray = taosArrayInit(1, sizeof(STagVal)))) { + goto _end; + } + + if (!(createTbArray = taosArrayInit(sz, POINTER_BYTES))) { + goto _end; + } + + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + // create table req + if (createTb) { + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + SVCreateTbReq *pCreateTbReq = NULL; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + taosArrayPush(createTbArray, &pCreateTbReq); + continue; + } + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // don't move to the end of loop as to destroy in the end of func when error occur + taosArrayPush(createTbArray, &pCreateTbReq); + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = taosStrdup((char *)tNameGetTableName(&name)); // taosStrdup(stbFullName); + + // set tag content + taosArrayClear(tagArray); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag *pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t *)pTag; + + // set tag name + SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + if (pDataBlock->info.parTbName[0]) { + pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName); + } else { + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + } + } + + // SSubmitTbData req + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + pDeleteReq->suid = suid; + pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); + continue; + } + + int32_t rows = pDataBlock->info.rows; + + SSubmitTbData tbData = {0}; + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + if (createTb) { + tbData.pCreateTbReq = taosArrayGetP(createTbArray, i); + if (tbData.pCreateTbReq) tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } + + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; ++j) { + taosArrayClear(pVals); + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn *pCol = &pTSchema->columns[k]; + SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void *data = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, data, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + } + } + SRow *pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); + } + + taosArrayPush(pReq->aSubmitTbData, &tbData); + } + + // encode + tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + if (TSDB_CODE_SUCCESS == terrno) { + SEncoder encoder; + len += sizeof(SSubmitReq2Msg); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SSubmitReq2Msg *)pBuf)->header.vgId = TD_VID(pVnode); + ((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len); + ((SSubmitReq2Msg *)pBuf)->version = htobe64(1); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); + if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + /*vError("failed to encode submit req since %s", terrstr());*/ + } + tEncoderClear(&encoder); + } +_end: + taosArrayDestroy(createTbArray); + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + if (terrno != 0) { + rpcFreeCont(pBuf); + taosArrayDestroy(pDeleteReq->deleteReqs); + return TSDB_CODE_FAILED; + } + if (ppData) *ppData = pBuf; + if (pLen) *pLen = len; + return TSDB_CODE_SUCCESS; } /** @@ -174,7 +378,7 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char STSmaStat *pTsmaStat = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { - terrno = TSDB_CODE_TSMA_INVALID_STAT; + terrno = TSDB_CODE_TSMA_INVALID_ENV; return TSDB_CODE_FAILED; } @@ -204,27 +408,32 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char } SBatchDeleteReq deleteReq = {0}; - SSubmitReq *pSubmitReq = - tqBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, - pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq); - // TODO deleteReq - taosArrayDestroy(deleteReq.deleteReqs); - + void *pSubmitReq = NULL; + int32_t contLen = 0; - if (!pSubmitReq) { - smaError("vgId:%d, failed to gen submit blk while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + if (smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, + pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, + &contLen) < 0) { + smaError("vgId:%d, failed to gen submit msg while tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); goto _err; } + // TODO deleteReq + taosArrayDestroy(deleteReq.deleteReqs); #if 0 - ASSERT(!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)); + if (!strncasecmp("td.tsma.rst.tb", pTsmaStat->pTSma->dstTbName, 14)) { + terrno = TSDB_CODE_APP_ERROR; + smaError("vgId:%d, tsma insert for smaIndex %" PRIi64 " failed since %s, %s", SMA_VID(pSma), indexUid, + pTsmaStat->pTSma->indexUid, tstrerror(terrno), pTsmaStat->pTSma->dstTbName); + goto _err; + } #endif SRpcMsg submitReqMsg = { .msgType = TDMT_VND_SUBMIT, .pCont = pSubmitReq, - .contLen = ntohl(pSubmitReq->length), + .contLen = contLen, }; if (tmsgPutToQueue(&pSma->pVnode->msgCb, WRITE_QUEUE, &submitReqMsg) < 0) { diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 4d09d690d684c68199f63165875216ea3685acad..7c538280e52ed127ff2815623a2afa00c126de7f 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -15,197 +15,72 @@ #include "sma.h" -// smaFileUtil ================ -#if 0 -#define TD_FILE_STATE_OK 0 -#define TD_FILE_STATE_BAD 1 +#define TD_QTASKINFO_FNAME_PREFIX "main.tdb" -#define TD_FILE_INIT_MAGIC 0xFFFFFFFF - -static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo); -static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo); - -static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { - int32_t tlen = 0; - - tlen += taosEncodeFixedU32(buf, pInfo->magic); - tlen += taosEncodeFixedU32(buf, pInfo->ftype); - tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedI64(buf, pInfo->fsize); - - return tlen; -} - -static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->magic)); - buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); - buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); - - return buf; -} - -int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte); - if (nwrite < nbyte) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nwrite; -} - -int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t loffset = taosLSeekFile(TD_TFILE_PFILE(pTFile), offset, whence); - if (loffset < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return loffset; -} - -int64_t tdGetTFileSize(STFile *pTFile, int64_t *size) { - ASSERT(TD_TFILE_OPENED(pTFile)); - return taosFStatFile(pTFile->pFile, size, NULL); -} - -int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte); - if (nread < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return nread; -} - -int32_t tdUpdateTFileHeader(STFile *pTFile) { - char buf[TD_FILE_HEAD_SIZE] = "\0"; - - if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { - return -1; - } - - void *ptr = buf; - tdEncodeTFInfo(&ptr, &(pTFile->info)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TD_FILE_HEAD_SIZE); - if (tdWriteTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { - return -1; - } - - return 0; -} - -int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) { - char buf[TD_FILE_HEAD_SIZE] = "\0"; - uint32_t _version; - - ASSERT(TD_TFILE_OPENED(pTFile)); - - if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { - return -1; - } - - if (tdReadTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)buf, TD_FILE_HEAD_SIZE)) { - terrno = TSDB_CODE_FILE_CORRUPTED; - return -1; - } - - void *pBuf = buf; - pBuf = tdDecodeTFInfo(pBuf, pInfo); - return 0; -} - -void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) { - pTFile->info.magic = taosCalcChecksum(pTFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +void tdRSmaQTaskInfoGetFileName(int32_t vgId, int64_t suid, int8_t level, int64_t version, char *outputName) { + tdRSmaGetFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); } -int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) { - ASSERT(TD_TFILE_OPENED(pTFile)); - - int64_t toffset; - - if ((toffset = tdSeekTFile(pTFile, 0, SEEK_END)) < 0) { - return -1; - } - -#if 0 - smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile), - toffset, nbyte, toffset + nbyte); -#endif - - ASSERT(pTFile->info.fsize == toffset); - - if (offset) { - *offset = toffset; - } - - if (tdWriteTFile(pTFile, buf, nbyte) < 0) { - return -1; - } - - pTFile->info.fsize += nbyte; - - return nbyte; +void tdRSmaQTaskInfoGetFullName(int32_t vgId, int64_t suid, int8_t level, int64_t version, const char *path, + char *outputName) { + tdRSmaGetFileName(vgId, path, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, suid, level, version, outputName); } -int32_t tdOpenTFile(STFile *pTFile, int flags) { - ASSERT(!TD_TFILE_OPENED(pTFile)); - - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), flags); - if (pTFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; +void tdRSmaQTaskInfoGetFullPath(int32_t vgId, int8_t level, const char *path, char *outputName) { + tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8, level); } -void tdCloseTFile(STFile *pTFile) { - if (TD_TFILE_OPENED(pTFile)) { - taosCloseFile(&pTFile->pFile); - TD_TFILE_SET_CLOSED(pTFile); - } +void tdRSmaQTaskInfoGetFullPathEx(int32_t vgId, tb_uid_t suid, int8_t level, const char *path, char *outputName) { + tdRSmaGetDirName(vgId, path, VNODE_RSMA_DIR, true, outputName); + int32_t rsmaLen = strlen(outputName); + snprintf(outputName + rsmaLen, TSDB_FILENAME_LEN - rsmaLen, "%" PRIi8 "%s%" PRIi64, level, TD_DIRSEP, suid); } -void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); } - -#endif - -void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, - char *outputName) { - if (version < 0) { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, - TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); +void tdRSmaGetFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t suid, + int8_t level, int64_t version, char *outputName) { + if (level >= 0 && suid > 0) { + if (version >= 0) { + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, pdname, + TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, + version); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s.%" PRIi64, TD_DIRSEP, + vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname, version); + } } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, - vgId, fname); + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", pdname, + TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s%" PRIi8 "%s%" PRIi64 "%s%s", TD_DIRSEP, vgId, + TD_DIRSEP, dname, TD_DIRSEP, level, TD_DIRSEP, suid, TD_DIRSEP, fname); + } } } else { - if (pdname) { - snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, - vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); + if (version >= 0) { + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP, + vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, + TD_DIRSEP, vgId, fname, version); + } } else { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname, - TD_DIRSEP, vgId, fname, version); + if (pdname) { + snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, + TD_DIRSEP, dname, TD_DIRSEP, vgId, fname); + } else { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, + TD_DIRSEP, vgId, fname); + } } } } -void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { +void tdRSmaGetDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) { if (pdname) { if (endWithSep) { snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP, @@ -223,81 +98,13 @@ void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool e } } -#if 0 -int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) { - TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK); - TD_TFILE_SET_CLOSED(pTFile); - - memset(&(pTFile->info), 0, sizeof(pTFile->info)); - pTFile->info.magic = TD_FILE_INIT_MAGIC; - - char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0}; - snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname); - int32_t tmpNameLen = strlen(tmpName) + 1; - pTFile->fname = taosMemoryMalloc(tmpNameLen); - if (!pTFile->fname) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - tstrncpy(pTFile->fname, tmpName, tmpNameLen); - - return 0; -} - -int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) { - ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pTFile->pFile == NULL) { - if (errno == ENOENT) { - // Try to create directory recursively - char *s = strdup(TD_TFILE_FULL_NAME(pTFile)); - if (taosMulMkDir(taosDirName(s)) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(s); - return -1; - } - taosMemoryFree(s); - pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (pTFile->pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - } - } - - if (!updateHeader) { - return 0; - } - - pTFile->info.fsize += TD_FILE_HEAD_SIZE; - pTFile->info.fver = 0; - - if (tdUpdateTFileHeader(pTFile) < 0) { - tdCloseTFile(pTFile); - tdRemoveTFile(pTFile); - return -1; - } - - return 0; -} - -int32_t tdRemoveTFile(STFile *pTFile) { - if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - }; - return 0; -} - -#endif - // smaXXXUtil ================ void *tdAcquireSmaRef(int32_t rsetId, int64_t refId) { void *pResult = taosAcquireRef(rsetId, refId); if (!pResult) { smaWarn("rsma acquire ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); } else { - smaDebug("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); + smaTrace("rsma acquire ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); } return pResult; } @@ -307,7 +114,7 @@ int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { smaWarn("rsma release ref for rsetId:%d refId:%" PRIi64 " failed since %s", rsetId, refId, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); + smaTrace("rsma release ref for rsetId:%d refId:%" PRIi64 " success", rsetId, refId); return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4f7f9d14eb1add07a8ab40a3a1f4b0d5ad32dfec..c9bbb30380a23be201eea3969b0b09beb580b0ed 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -78,7 +78,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pTq->path = strdup(path); + pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; @@ -169,19 +169,17 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { SMqDataRsp* pRsp = &pPushEntry->dataRsp; - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - ASSERT(!pRsp->withSchema); - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(!pRsp->withSchema); + A(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - /*if (pRsp->blockNum > 0) {*/ - /*ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version);*/ - /*} else {*/ - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); - /*}*/ + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } +#endif int32_t len = 0; int32_t code = 0; @@ -226,19 +224,21 @@ int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { } int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); - ASSERT(!pRsp->withSchema); - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(!pRsp->withSchema); + A(taosArrayGetSize(pRsp->blockSchema) == 0); if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); + A(pRsp->rspOffset.version >= pRsp->reqOffset.version); } } +#endif int32_t len = 0; int32_t code = 0; @@ -282,22 +282,24 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con } int32_t tqSendTaosxRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const STaosxRsp* pRsp) { - ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +#if 0 + A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); if (pRsp->withSchema) { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); + A(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); } else { - ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + A(taosArrayGetSize(pRsp->blockSchema) == 0); } if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { if (pRsp->blockNum > 0) { - ASSERT(pRsp->rspOffset.version > pRsp->reqOffset.version); + A(pRsp->rspOffset.version > pRsp->reqOffset.version); } else { - ASSERT(pRsp->rspOffset.version >= pRsp->reqOffset.version); + A(pRsp->rspOffset.version >= pRsp->reqOffset.version); } } +#endif int32_t len = 0; int32_t code = 0; @@ -348,10 +350,9 @@ static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOf int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { STqOffset offset = {0}; - SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*) msg, msgLen); + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); if (tDecodeSTqOffset(&decoder, &offset) < 0) { - ASSERT(0); return -1; } @@ -378,7 +379,6 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t // save the new offset value if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { - ASSERT(0); return -1; } @@ -440,7 +440,7 @@ static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t su } #endif - ASSERT(subType == TOPIC_SUB_TYPE__COLUMN); + /*A(subType == TOPIC_SUB_TYPE__COLUMN);*/ pRsp->withSchema = false; return 0; @@ -571,7 +571,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); // lock taosWLockLatch(&pTq->pushLock); - tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew); + if (tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew) < 0) { + return -1; + } #if 1 // till now, all data has been rsp to consumer, new data needs to push client once arrived. @@ -611,7 +613,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // for taosx - ASSERT(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN); + /*A(pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN);*/ SMqMetaRsp metaRsp = {0}; @@ -619,7 +621,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqInitTaosxRsp(&taosxRsp, &req); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { - tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew) < 0) { + return -1; + } if (metaRsp.metaRspLen > 0) { if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { @@ -685,9 +689,15 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { - SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - - if (tqTaosxScanLog(pTq, pHandle, pCont, &taosxRsp) < 0) { + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), + .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), + .ver = pHead->version, + }; + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp) < 0) { + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, TD_VID(pTq->pVnode), + req.subKey); + return -1; } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); @@ -702,9 +712,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } } else { - ASSERT(pHandle->fetchMeta); - ASSERT(IS_META_MSG(pHead->msgType)); - tqDebug("fetch meta msg, ver:%" PRId64 ", type:%d", pHead->version, pHead->msgType); + /*A(pHandle->fetchMeta);*/ + /*A(IS_META_MSG(pHead->msgType));*/ + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; @@ -731,7 +741,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; - tqDebug("vgId:%d, delete sub: %s", pTq->pVnode->config.vgId, pReq->subKey); + tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); taosWLockLatch(&pTq->pushLock); int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); @@ -742,6 +752,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { + // walCloseRef(pHandle->pWalReader->pWal, pHandle->pRef->refId); if (pHandle->pRef) { walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); } @@ -765,7 +776,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { STqCheckInfo info = {0}; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*) msg, msgLen); + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); if (tDecodeSTqCheckInfo(&decoder, &info) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -798,6 +809,9 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock + + tqDebug("vgId:%d, tq process sub req %s", pTq->pVnode->config.vgId, req.subKey); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { @@ -881,12 +895,14 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg return -1; } } else { - /*ASSERT(pExec->consumerId == req.oldConsumerId);*/ // TODO handle qmsg and exec modification atomic_store_32(&pHandle->epoch, -1); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); taosMemoryFree(req.qmsg); + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + qStreamCloseTsdbReader(pHandle->execHandle.task); + } if (tqMetaSaveHandle(pTq, req.subKey, pHandle) < 0) { return -1; } @@ -897,9 +913,11 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { +#if 0 if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); + A(taosArrayGetSize(pTask->childEpInfo) != 0); } +#endif pTask->refCnt = 1; pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; @@ -936,7 +954,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { .pStateBackend = pTask->pState, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); - ASSERT(pTask->exec.executor); + if (pTask->exec.executor == NULL) { + return -1; + } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); @@ -949,7 +969,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { .pStateBackend = pTask->pState, }; pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle); - ASSERT(pTask->exec.executor); + if (pTask->exec.executor == NULL) { + return -1; + } } // sink @@ -959,13 +981,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->smaSink.smaSink = smaHandleRes; } else if (pTask->outputType == TASK_OUTPUT__TABLE) { pTask->tbSink.vnode = pTq->pVnode; - pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline; + pTask->tbSink.tbSinkFunc = tqSinkToTablePipeline2; - ASSERT(pTask->tbSink.pSchemaWrapper); - ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); + /*A(pTask->tbSink.pSchemaWrapper);*/ + /*A(pTask->tbSink.pSchemaWrapper->pSchema);*/ pTask->tbSink.pTSchema = - tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); + tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); ASSERT(pTask->tbSink.pTSchema); } @@ -982,7 +1004,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); SStreamTaskCheckReq req; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*) msgBody, msgLen); + tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeSStreamTaskCheckReq(&decoder, &req); tDecoderClear(&decoder); int32_t taskId = req.downstreamTaskId; @@ -1012,7 +1034,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { int32_t len; tEncodeSize(tEncodeSStreamTaskCheckRsp, &rsp, len, code); if (code < 0) { - tqDebug("tq encode stream check rsp error"); + tqError("unable to encode rsp %d", __LINE__); return -1; } @@ -1111,12 +1133,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { if (pTask == NULL) { return -1; } - ASSERT(pReq->taskId == pTask->taskId); // check param int64_t fillVer1 = pTask->startVer; if (fillVer1 <= 0) { - ASSERT(0); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return -1; } @@ -1223,7 +1243,7 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { SStreamRecoverFinishReq req; SDecoder decoder; - tDecoderInit(&decoder, (uint8_t*) msg, msgLen); + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); tDecodeSStreamRecoverFinishReq(&decoder, &req); tDecoderClear(&decoder); @@ -1328,7 +1348,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } int32_t ref = atomic_sub_fetch_32(pRef, 1); - ASSERT(ref >= 0); + /*A(ref >= 0);*/ if (ref == 0) { blockDataDestroy(pDelBlock); taosMemoryFree(pRef); @@ -1363,12 +1383,12 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit* pSubmit = NULL; +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { + void* pIter = NULL; + bool failed = false; + SStreamDataSubmit2* pSubmit = NULL; - pSubmit = streamDataSubmitNew(pReq); + pSubmit = streamDataSubmitNew(submit); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); @@ -1388,7 +1408,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 093186ebbb188841bf0fa012be4dd59f8d0b5a1b..e92da7668ad85c8d42d8d238222d830cad54fce3 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,13 +29,12 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); taosArrayPush(pRsp->blockData, &buf); return 0; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader->pSchemaWrapper); if (pSW == NULL) { return -1; @@ -44,7 +43,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRs return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t n) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, int32_t n) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); // TODO add reference to gurantee success @@ -53,7 +52,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i return -1; } for (int32_t i = 0; i < n; i++) { - char* tbName = strdup(mr.me.name); + char* tbName = taosStrdup(mr.me.name); taosArrayPush(pRsp->blockTbName, &tbName); } metaReaderClear(&mr); @@ -62,7 +61,6 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { const STqExecHandle* pExec = &pHandle->execHandle; - ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->task; @@ -87,7 +85,8 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs uint64_t ts = 0; tqDebug("vgId:%d, tmq task start to execute", pTq->pVnode->config.vgId); if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + return -1; } tqDebug("vgId:%d, tmq task executed, get %p", pTq->pVnode->config.vgId, pDataBlock); @@ -105,19 +104,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - ASSERT(0); return -1; } - ASSERT(pRsp->rspOffset.type != 0); - if (pRsp->withTbName) { - if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, 1); - } else { - pRsp->withTbName = false; - } + if (pRsp->rspOffset.type == 0) { + tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, + pRsp->rspOffset.uid, pRsp->rspOffset.version); + return -1; } + + ASSERT(pRsp->withTbName == false); ASSERT(pRsp->withSchema == false); return 0; @@ -148,26 +144,26 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta uint64_t ts = 0; tqDebug("tmqsnap task start to execute"); if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); + return -1; } tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL) { if (pRsp->withTbName) { - int64_t uid = 0; if (pOffset->type == TMQ_OFFSET__LOG) { - uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, 1) < 0) { + int64_t uid = pExec->pExecReader->lastBlkUid; + if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { continue; } } else { - char* tbName = strdup(qExtractTbnameFromTask(task)); + char* tbName = taosStrdup(qExtractTbnameFromTask(task)); taosArrayPush(pRsp->blockTbName, &tbName); } } if (pRsp->withSchema) { if (pOffset->type == TMQ_OFFSET__LOG) { - tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); + tqAddBlockSchemaToRsp(pExec, pRsp); } else { SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); taosArrayPush(pRsp->blockSchema, &pSW); @@ -215,25 +211,29 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta break; } - if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { - ASSERT(0); + qStreamExtractOffset(task, &pRsp->rspOffset); + + if (pRsp->rspOffset.type == 0) { + tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, + pRsp->rspOffset.uid, pRsp->rspOffset.version); + return -1; } - ASSERT(pRsp->rspOffset.type != 0); return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp* pRsp) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp) { STqExecHandle* pExec = &pHandle->execHandle; - ASSERT(pExec->subType != TOPIC_SUB_TYPE__COLUMN); + /*A(pExec->subType != TOPIC_SUB_TYPE__COLUMN);*/ SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pSchemas = taosArrayInit(0, sizeof(void*)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlock(pReader)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock2(pReader)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ @@ -241,12 +241,14 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + SSubmitTbData* pSubmitTbDataRet = NULL; + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { + /*int64_t uid = pExec->pExecReader->msgIter.uid;*/ + int64_t uid = pExec->pExecReader->lastBlkUid; + if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); @@ -254,20 +256,33 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } - if (pHandle->fetchMeta) { - SSubmitBlk* pBlk = pReader->pBlock; - int32_t schemaLen = htonl(pBlk->schemaLen); - if (schemaLen > 0) { - if (pRsp->createTableNum == 0) { - pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); - } - void* createReq = taosMemoryCalloc(1, schemaLen); - memcpy(createReq, pBlk->data, schemaLen); - taosArrayPush(pRsp->createTableLen, &schemaLen); - taosArrayPush(pRsp->createTableReq, &createReq); - pRsp->createTableNum++; + if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + } + + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); + if (TSDB_CODE_SUCCESS != code) { + continue; } + void* createReq = taosMemoryCalloc(1, len); + SEncoder encoder = {0}; + tEncoderInit(&encoder, createReq, len); + code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq); + if (code < 0) { + tEncoderClear(&encoder); + taosMemoryFree(createReq); + continue; + } + + taosArrayPush(pRsp->createTableLen, &len); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; + + tEncoderClear(&encoder); } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { SSDataBlock* pBlock = taosArrayGet(pBlocks, i); @@ -281,20 +296,22 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pExecReader; - tqReaderSetDataMsg(pReader, pReq, 0); - while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { + /*tqReaderSetDataMsg(pReader, pReq, 0);*/ + tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { /*SSDataBlock block = {0};*/ /*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/ /*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/ /*}*/ taosArrayClear(pBlocks); taosArrayClear(pSchemas); - if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas) < 0) { + SSubmitTbData* pSubmitTbDataRet = NULL; + if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->msgIter.uid; - if (tqAddTbNameToRsp(pTq, uid, (SMqDataRsp*)pRsp, taosArrayGetSize(pBlocks)) < 0) { + int64_t uid = pExec->pExecReader->lastBlkUid; + if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); @@ -302,20 +319,33 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } - if (pHandle->fetchMeta) { - SSubmitBlk* pBlk = pReader->pBlock; - int32_t schemaLen = htonl(pBlk->schemaLen); - if (schemaLen > 0) { - if (pRsp->createTableNum == 0) { - pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); - } - void* createReq = taosMemoryCalloc(1, schemaLen); - memcpy(createReq, pBlk->data, schemaLen); - taosArrayPush(pRsp->createTableLen, &schemaLen); - taosArrayPush(pRsp->createTableReq, &createReq); - pRsp->createTableNum++; + if (pHandle->fetchMeta && pSubmitTbDataRet->pCreateTbReq != NULL) { + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + } + + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + tEncodeSize(tEncodeSVCreateTbReq, pSubmitTbDataRet->pCreateTbReq, len, code); + if (TSDB_CODE_SUCCESS != code) { + continue; + } + void* createReq = taosMemoryCalloc(1, len); + SEncoder encoder = {0}; + tEncoderInit(&encoder, createReq, len); + code = tEncodeSVCreateTbReq(&encoder, pSubmitTbDataRet->pCreateTbReq); + if (code < 0) { + tEncoderClear(&encoder); + taosMemoryFree(createReq); + continue; } + + taosArrayPush(pRsp->createTableLen, &len); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; + + tEncoderClear(&encoder); } /*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/ /*pTq->pVnode->config.tsdbCfg.precision);*/ @@ -333,13 +363,11 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp } } } - taosArrayDestroy(pBlocks); taosArrayDestroy(pSchemas); - - if (pRsp->blockNum == 0) { - return -1; - } +// if (pRsp->blockNum == 0) { +// return -1; +// } return 0; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 095251ab73fdcb22da8c8181cb3bb5fac5cc99df..12fd15c63a5088a8da2ceb4de9b57f653976bc8a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -132,19 +132,19 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); + return -1; } if (tdbTbDelete(pTq->pCheckStore, key, (int)strlen(key), txn) < 0) { - /*ASSERT(0);*/ + tqWarn("vgId:%d, tq try delete checkinfo failed %s", pTq->pVnode->config.vgId, key); } if (tdbCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } return 0; @@ -153,7 +153,6 @@ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key) { int32_t tqMetaRestoreCheckInfo(STQ* pTq) { TBC* pCur = NULL; if (tdbTbcOpen(pTq->pCheckStore, &pCur, NULL) < 0) { - ASSERT(0); return -1; } @@ -194,6 +193,9 @@ int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle) { int32_t code; int32_t vlen; tEncodeSize(tEncodeSTqHandle, pHandle, vlen, code); + if (code < 0) { + return -1; + } tqDebug("tq save %s(%d) handle consumer:0x%" PRIx64 "epoch:%d vgId:%d", pHandle->subKey, (int32_t)strlen(pHandle->subKey), pHandle->consumerId, pHandle->epoch, TD_VID(pTq->pVnode)); @@ -249,19 +251,18 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { if (tdbBegin(pTq->pMetaDB, &txn, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { - ASSERT(0); + return -1; } if (tdbTbDelete(pTq->pExecStore, key, (int)strlen(key), txn) < 0) { - /*ASSERT(0);*/ } if (tdbCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } if (tdbPostCommit(pTq->pMetaDB, txn) < 0) { - ASSERT(0); + return -1; } return 0; @@ -270,7 +271,6 @@ int32_t tqMetaDeleteHandle(STQ* pTq, const char* key) { int32_t tqMetaRestoreHandle(STQ* pTq) { TBC* pCur = NULL; if (tdbTbcOpen(pTq->pExecStore, &pCur, NULL) < 0) { - ASSERT(0); return -1; } @@ -290,7 +290,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { handle.pRef = walOpenRef(pTq->pVnode->pWal); if (handle.pRef == NULL) { - continue; + return -1; } walRefVer(handle.pRef, handle.snapshotVer); @@ -305,12 +305,19 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { if (handle.execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { handle.execHandle.task = qCreateQueueExecTaskInfo(handle.execHandle.execCol.qmsg, &reader, &handle.execHandle.numOfCols, NULL); - ASSERT(handle.execHandle.task); + if (handle.execHandle.task == NULL) { + tqError("cannot create exec task for %s", handle.subKey); + return -1; + } void* scanner = NULL; qExtractStreamScanner(handle.execHandle.task, &scanner); - ASSERT(scanner); + if (scanner == NULL) { + tqError("cannot extract stream scanner for %s", handle.subKey); + } handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - ASSERT(handle.execHandle.pExecReader); + if (handle.execHandle.pExecReader == NULL) { + tqError("cannot extract exec reader for %s", handle.subKey); + } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 338f9d6c2424d65e4dd289c728b72c01f0c971d7..66d1ac2c7e530e4b8c732deaaa83afc6191b5e44 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -40,8 +40,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { if (code == 0) { break; } else { - ASSERT(0); - // TODO handle error + return -1; } } int32_t size = htonl(head.size); @@ -102,6 +101,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); if (tqOffsetRestoreFromFile(pStore, fname) < 0) { taosMemoryFree(fname); + taosMemoryFree(pStore); return NULL; } taosMemoryFree(fname); @@ -153,9 +153,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { int32_t bodyLen; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); - ASSERT(code == 0); if (code < 0) { - ASSERT(0); taosHashCancelIterate(pStore->pHash, pIter); return -1; } @@ -171,7 +169,6 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { // write file int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { - ASSERT(0); tqError("write offset incomplete, len %d, write len %" PRId64, bodyLen, writeLen); taosHashCancelIterate(pStore->pHash, pIter); taosMemoryFree(buf); diff --git a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c index b63ff8af1d623aa2017ce5768a80e2dbf783d5a4..a4428aed4368fec9c96a2faae5fabd33cd8eb8f4 100644 --- a/source/dnode/vnode/src/tq/tqOffsetSnapshot.c +++ b/source/dnode/vnode/src/tq/tqOffsetSnapshot.c @@ -61,19 +61,23 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) { int64_t sz = 0; if (taosStatFile(fname, &sz, NULL) < 0) { - ASSERT(0); + taosCloseFile(&pFile); + taosMemoryFree(fname); + return -1; } taosMemoryFree(fname); SSnapDataHdr* buf = taosMemoryCalloc(1, sz + sizeof(SSnapDataHdr)); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + taosCloseFile(&pFile); return terrno; } void* abuf = POINTER_SHIFT(buf, sizeof(SSnapDataHdr)); int64_t contLen = taosReadFile(pFile, abuf, sz); if (contLen != sz) { - ASSERT(0); + taosCloseFile(&pFile); + taosMemoryFree(buf); return -1; } buf->size = sz; @@ -122,14 +126,17 @@ int32_t tqOffsetWriterClose(STqOffsetWriter** ppWriter, int8_t rollback) { if (rollback) { if (taosRemoveFile(pWriter->fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } } else { if (taosRenameFile(pWriter->fname, fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } if (tqOffsetRestoreFromFile(pTq->pOffsetStore, fname) < 0) { - ASSERT(0); + taosMemoryFree(fname); + return -1; } } taosMemoryFree(fname); @@ -146,14 +153,13 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa TdFilePtr pFile = taosOpenFile(pWriter->fname, TD_FILE_CREATE | TD_FILE_WRITE); SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; int64_t size = pHdr->size; - ASSERT(size == nData - sizeof(SSnapDataHdr)); if (pFile) { int64_t contLen = taosWriteFile(pFile, pHdr->data, size); if (contLen != size) { - ASSERT(0); + taosCloseFile(&pFile); + return -1; } } else { - ASSERT(0); return -1; } return 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 7a356238a0acc27c4530f6829e9cc09d910b0c83..e74726227794b2aab7072e81203e1747f2eb4ea7 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -25,9 +25,7 @@ void tqTmrRspFunc(void* param, void* tmrId) { static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) { SStreamDataSubmit* pSubmit = *ppSubmit; while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { - /*ASSERT(0);*/ } // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); @@ -160,8 +158,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ if (msgType == TDMT_VND_SUBMIT) { tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); } else { - // TODO - ASSERT(0); + tqError("tq push unexpected msg type %d", msgType); } if (rsp.blockNum == 0) { @@ -169,9 +166,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ continue; } - ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); - ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); - rsp.rspOffset = fetchOffset; int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); @@ -213,7 +207,11 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - tqDebug("vgId:%d tq push msg version:%" PRId64 " type: %s", pTq->pVnode->config.vgId, ver, TMSG_INFO(msgType)); + void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); + int32_t len = msgLen - sizeof(SSubmitReq2Msg); + + tqDebug("vgId:%d tq push msg version:%" PRId64 " type: %s, p head %p, p body %p, len %d", pTq->pVnode->config.vgId, ver, + TMSG_INFO(msgType), msg, pReq, len); if (msgType == TDMT_VND_SUBMIT) { // lock push mgr to avoid potential msg lost @@ -223,7 +221,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) tqDebug("vgId:%d, push handle num %d", pTq->pVnode->config.vgId, taosHashGetSize(pTq->pPushMgr)); SArray* cachedKeys = taosArrayInit(0, sizeof(void*)); SArray* cachedKeyLens = taosArrayInit(0, sizeof(size_t)); - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); @@ -231,9 +229,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) taosArrayDestroy(cachedKeyLens); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); void* pIter = NULL; while (1) { @@ -259,14 +255,19 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) SMqDataRsp* pRsp = &pPushEntry->dataRsp; // prepare scan mem data - qStreamScanMemData(task, pReq); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + qStreamSetScanMemData(task, submit); // exec while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; if (qExecTask(task, &pDataBlock, &ts) < 0) { - ASSERT(0); + tqDebug("vgId:%d, tq exec error since %s", pTq->pVnode->config.vgId, terrstr()); } if (pDataBlock == NULL) { @@ -285,7 +286,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) // remove from hash size_t kLen; void* key = taosHashGetKey(pIter, &kLen); - void* keyCopy = taosMemoryMalloc(kLen); + void* keyCopy = taosMemoryCalloc(1, kLen + 1); memcpy(keyCopy, key, kLen); taosArrayPush(cachedKeys, &keyCopy); @@ -299,7 +300,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* key = taosArrayGetP(cachedKeys, i); size_t kLen = *(size_t*)taosArrayGet(cachedKeyLens, i); if (taosHashRemove(pTq->pPushMgr, key, kLen) != 0) { - ASSERT(0); + tqError("vgId:%d, tq push hash remove key error, key: %s", pTq->pVnode->config.vgId, (char*)key); } } taosArrayDestroyP(cachedKeys, (FDelete)taosMemoryFree); @@ -313,17 +314,22 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - void* data = taosMemoryMalloc(msgLen); + void* data = taosMemoryMalloc(len); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to copy data for stream since out of memory"); return -1; } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; + memcpy(data, pReq, len); + SPackedData submit = { + .msgStr = data, + .msgLen = len, + .ver = ver, + }; + + tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); - tqProcessSubmitReq(pTq, data, ver); + tqProcessSubmitReq(pTq, submit); } if (msgType == TDMT_VND_DELETE) { tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 631652f7fa774b651b7060d62edc852ae76d6150..2c8f20d8cd96432b943d4db208e17bc843dac836 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -176,8 +176,6 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { goto end; } realTbSuid = req.suid; - } else { - ASSERT(0); } end: @@ -206,7 +204,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -220,7 +217,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea if (IS_META_MSG(pHead->msgType)) { code = walFetchBody(pHandle->pWalReader, ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -238,7 +234,6 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea } code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead); if (code < 0) { - ASSERT(0); *fetchOffset = offset; code = -1; goto END; @@ -252,7 +247,7 @@ END: } STqReader* tqOpenReader(SVnode* pVnode) { - STqReader* pReader = taosMemoryMalloc(sizeof(STqReader)); + STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; } @@ -264,7 +259,7 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - pReader->pMsg = NULL; + /*pReader->pMsg = NULL;*/ pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; @@ -303,7 +298,7 @@ int32_t tqSeekVer(STqReader* pReader, int64_t ver) { } int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->pMsg != NULL; + bool fromProcessedMsg = pReader->msg2.msgStr != NULL; while (1) { if (!fromProcessedMsg) { @@ -314,10 +309,11 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; tqDebug("return offset %" PRId64 ", no more valid", ret->offset.version); - ASSERT(ret->offset.version >= 0); return -1; } - void* body = pReader->pWalReader->pHead->head.body; + void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pReader->pWalReader->pHead->head.version; #if 0 if (pReader->pWalReader->pHead->head.msgType != TDMT_VND_SUBMIT) { // TODO do filter @@ -326,18 +322,18 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { return 0; } else { #endif - tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version); + tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + /*tqReaderSetDataMsg(pReader, body, pReader->pWalReader->pHead->head.version);*/ #if 0 } #endif } - while (tqNextDataBlock(pReader)) { + while (tqNextDataBlock2(pReader)) { // TODO mem free memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(&ret->data, pReader); + int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); if (code != 0 || ret->data.info.rows == 0) { - ASSERT(0); continue; } ret->fetchType = FETCH_TYPE__DATA; @@ -348,7 +344,6 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { if (fromProcessedMsg) { ret->offset.type = TMQ_OFFSET__LOG; ret->offset.version = pReader->ver; - ASSERT(pReader->ver >= 0); ret->fetchType = FETCH_TYPE__SEP; tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); return 0; @@ -356,6 +351,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { } } +#if 0 int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { pReader->pMsg = pMsg; @@ -372,7 +368,33 @@ int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t v memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); return 0; } +#endif + +int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { + ASSERT(pReader->msg2.msgStr == NULL); + ASSERT(msgStr); + ASSERT(msgLen); + ASSERT(ver >= 0); + pReader->msg2.msgStr = msgStr; + pReader->msg2.msgLen = msgLen; + pReader->msg2.ver = ver; + pReader->ver = ver; + + tqDebug("tq reader set msg %p %d", msgStr, msgLen); + + if (pReader->setMsg == 0) { + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); + pReader->setMsg = 1; + } + return 0; +} +#if 0 bool tqNextDataBlock(STqReader* pReader) { if (pReader->pMsg == NULL) return false; while (1) { @@ -396,6 +418,60 @@ bool tqNextDataBlock(STqReader* pReader) { } return false; } +#endif + +bool tqNextDataBlock2(STqReader* pReader) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + ASSERT(pSubmitTbData->uid); + + if (pReader->tbIdHash == NULL) return true; + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} + +bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { + if (pReader->msg2.msgStr == NULL) return false; + ASSERT(pReader->setMsg == 1); + + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < blockSz) { + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (filterOutUids == NULL) return true; + + void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret == NULL) { + return true; + } + pReader->nextBlk++; + } + + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->setMsg = 0; + pReader->nextBlk = 0; + pReader->msg2.msgStr = NULL; + + return false; +} int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrapper* pSrc, char* mask) { int32_t code; @@ -426,6 +502,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } +#if 0 bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -433,7 +510,6 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { } if (pHandle->pBlock == NULL) return false; - ASSERT(pHandle->tbIdHash == NULL); void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); if (ret == NULL) { return true; @@ -442,6 +518,252 @@ bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { return false; } +int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { + // + int32_t sversion = htonl(pReader->pBlock->sversion); + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || + pReader->cachedSchemaSuid != pReader->msgIter.suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + } + return 0; +} +#endif + +int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + ASSERT(pReader->nextBlk < blockSz); + + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg2.ver; + + if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + + int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); + + if (colNumNeed == 0) { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { + goto FAIL; + } + colMeta++; + colNeed++; + } + } + } + + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + ASSERT(sourceIdx < numOfCols); + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + ASSERT(pCol->nVal == numOfRows); + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, i, &colVal); +#if 0 + void* val = NULL; + if (IS_STR_DATA_TYPE(colVal.type)) { + val = colVal.value.pData; + } else { + val = &colVal.value.val; + } + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } +#endif + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataSetNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + ASSERT(0); + } + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + int32_t sourceIdx = 0; + + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + ASSERT(sourceIdx < pTschema->numOfCols); + + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataSetNULL(pColData, i); + } + /*val = colVal.value.pData;*/ + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + + sourceIdx++; + break; + } else { + ASSERT(0); + } + } + } + } + } + } + + return 0; + +FAIL: + blockDataFreeRes(pBlock); + return -1; +} + +#if 0 int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { // TODO: cache multiple schema int32_t sversion = htonl(pReader->pBlock->sversion); @@ -452,7 +774,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -463,7 +784,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->msgIter.uid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -566,7 +886,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (pReader->pSchema == NULL) { tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -577,7 +896,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (pReader->pSchemaWrapper == NULL) { tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->msgIter.uid, pReader->cachedSchemaVer); - /*ASSERT(0);*/ pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; @@ -686,6 +1004,262 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas taosMemoryFree(assigned); return 0; +FAIL: + taosMemoryFree(assigned); + return -1; +} +#endif + +int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { + tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + pReader->nextBlk++; + + if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + int32_t sversion = pSubmitTbData->sver; + int64_t suid = pSubmitTbData->suid; + int64_t uid = pSubmitTbData->uid; + pReader->lastBlkUid = uid; + + taosMemoryFree(pReader->pSchema); + pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchema == NULL) { + tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 + "), version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); + if (pReader->pSchemaWrapper == NULL) { + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + pReader->cachedSchemaSuid = 0; + terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; + return -1; + } + + STSchema* pTschema = pReader->pSchema; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + int32_t numOfRows = 0; + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + SColData* pCol = taosArrayGet(pCols, 0); + numOfRows = pCol->nVal; + } else { + SArray* pRows = pSubmitTbData->aRowP; + numOfRows = taosArrayGetSize(pRows); + } + + int32_t curRow = 0; + int32_t lastRow = 0; + char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); + if (assigned == NULL) return -1; + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + for (int32_t i = 0; i < numOfRows; i++) { + bool buildNew = false; + + for (int32_t j = 0; j < numOfCols; j++) { + SColData* pCol = taosArrayGet(pCols, j); + SColVal colVal; + tColDataGetValue(pCol, i, &colVal); + if (curRow == 0) { + assigned[j] = !COL_VAL_IS_NONE(&colVal); + buildNew = true; + } else { + bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); + if (currentRowAssigned != assigned[j]) { + assigned[j] = currentRowAssigned; + buildNew = true; + } + } + } + + if (buildNew) { + if (taosArrayGetSize(blocks) > 0) { + SSDataBlock* pLastBlock = taosArrayGetLast(blocks); + pLastBlock->info.rows = curRow - lastRow; + lastRow = curRow; + } + + SSDataBlock block = {0}; + SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pSW == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) { + blockDataFreeRes(&block); + tDeleteSSchemaWrapper(pSW); + goto FAIL; + } + tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, + (int32_t)taosArrayGetSize(block.pDataBlock)); + + block.info.id.uid = uid; + block.info.version = pReader->msg2.ver; + if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + blockDataFreeRes(&block); + tDeleteSSchemaWrapper(pSW); + goto FAIL; + } + taosArrayPush(blocks, &block); + taosArrayPush(schemas, &pSW); + } + + SSDataBlock* pBlock = taosArrayGetLast(blocks); + + tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId, + (int32_t)taosArrayGetSize(blocks)); + + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + int32_t colActual = blockDataGetNumOfCols(pBlock); + while (targetIdx < colActual) { + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + tColDataGetValue(pCol, i, &colVal); + + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataSetNULL(pColData, curRow - lastRow); + } + } else { + if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + sourceIdx++; + targetIdx++; + } + } + + curRow++; + } + } else { + SArray* pRows = pSubmitTbData->aRowP; + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); + bool buildNew = false; + + for (int32_t j = 0; j < pTschema->numOfCols; j++) { + SColVal colVal; + tRowGet(pRow, pTschema, j, &colVal); + if (curRow == 0) { + assigned[j] = !COL_VAL_IS_NONE(&colVal); + buildNew = true; + } else { + bool currentRowAssigned = !COL_VAL_IS_NONE(&colVal); + if (currentRowAssigned != assigned[j]) { + assigned[j] = currentRowAssigned; + buildNew = true; + } + } + } + + if (buildNew) { + if (taosArrayGetSize(blocks) > 0) { + SSDataBlock* pLastBlock = taosArrayGetLast(blocks); + pLastBlock->info.rows = curRow - lastRow; + lastRow = curRow; + } + + SSDataBlock block = {0}; + SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pSW == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) { + blockDataFreeRes(&block); + tDeleteSSchemaWrapper(pSW); + goto FAIL; + } + tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, + (int32_t)taosArrayGetSize(block.pDataBlock)); + + block.info.id.uid = uid; + block.info.version = pReader->msg2.ver; + if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + blockDataFreeRes(&block); + tDeleteSSchemaWrapper(pSW); + goto FAIL; + } + taosArrayPush(blocks, &block); + taosArrayPush(schemas, &pSW); + } + + SSDataBlock* pBlock = taosArrayGetLast(blocks); + + tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId, + (int32_t)taosArrayGetSize(blocks)); + + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + int32_t colActual = blockDataGetNumOfCols(pBlock); + while (targetIdx < colActual) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + tRowGet(pRow, pTschema, sourceIdx, &colVal); + + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + } else if (colVal.cid == pColData->info.colId) { + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2]; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, curRow - lastRow, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataSetNULL(pColData, curRow - lastRow); + } + } else { + if (colDataAppend(pColData, curRow - lastRow, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + sourceIdx++; + targetIdx++; + } + } + curRow++; + } + } + + SSDataBlock* pLastBlock = taosArrayGetLast(blocks); + pLastBlock->info.rows = curRow - lastRow; + + taosMemoryFree(assigned); + return 0; + FAIL: taosMemoryFree(assigned); return -1; @@ -731,8 +1305,6 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { } int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { - ASSERT(pReader->tbIdHash != NULL); - for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); taosHashRemove(pReader->tbIdHash, pKey, sizeof(int64_t)); @@ -752,7 +1324,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { STqHandle* pExec = (STqHandle*)pIter; if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); - ASSERT(code == 0); + if (code != 0) { + tqError("update qualified table error for %s", pExec->subKey); + continue; + } } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); @@ -771,7 +1346,7 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { int32_t code = metaGetTableEntryByUidCache(&mr, *id); if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); + tqError("failed to get table meta, uid:%" PRIu64 " code:%s", *id, tstrerror(terrno)); continue; } @@ -790,10 +1365,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } taosArrayDestroy(qa); } else { - // TODO handle delete table from stb + tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList); } - } else { - ASSERT(0); } } while (1) { @@ -802,7 +1375,10 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); - ASSERT(code == 0); + if (code != 0) { + tqError("update qualified table error for stream task %d", pTask->taskId); + continue; + } } } return 0; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 6d474d5aa1360e9d7425b8b6926439ee1aafd160..56684b691b3f76daf4fd7b346eaa6fa286e32461 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -19,7 +19,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq) { - ASSERT(pDataBlock->info.type == STREAM_DELETE_RESULT); int32_t totRow = pDataBlock->info.rows; SColumnInfoData* pStartTsCol = taosArrayGet(pDataBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDataBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -72,234 +71,6 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl return 0; } -SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pTSchema, - SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, - SBatchDeleteReq* pDeleteReq) { - SSubmitReq* ret = NULL; - SArray* schemaReqs = NULL; - SArray* schemaReqSz = NULL; - SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); - if (!tagArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - int32_t sz = taosArrayGetSize(pBlocks); - - if (createTb) { - schemaReqs = taosArrayInit(sz, sizeof(void*)); - schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - int32_t padding1 = 0; - void* padding2 = NULL; - taosArrayPush(schemaReqSz, &padding1); - taosArrayPush(schemaReqs, &padding2); - continue; - } - - // STag* pTag = NULL; - // taosArrayClear(tagArray); - // SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - // for(int j = 0; j < pTagSchemaWrapper->nCols; j++){ - // STagVal tagVal = { - // .cid = pTagSchemaWrapper->pSchema[j].colId, - // .type = pTagSchemaWrapper->pSchema[j].type, - // .i64 = (int64_t)pDataBlock->info.id.groupId, - // }; - // taosArrayPush(tagArray, &tagVal); - // taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name); - // } - // - // tTagNew(tagArray, 1, false, &pTag); - // if (pTag == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - // taosArrayDestroy(tagArray); - // taosArrayDestroy(tagName); - // return NULL; - // } - - SVCreateTbReq createTbReq = {0}; - - // set const - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - // set super table name - SName name = {0}; - tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); - - // set tag content - taosArrayClear(tagArray); - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.id.groupId, - }; - taosArrayPush(tagArray, &tagVal); - createTbReq.ctb.tagNum = taosArrayGetSize(tagArray); - - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - // set tag name - SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); - char tagNameStr[TSDB_COL_NAME_LEN] = {0}; - strcpy(tagNameStr, "group_id"); - taosArrayPush(tagName, tagNameStr); - createTbReq.ctb.tagName = tagName; - - // set table name - if (pDataBlock->info.parTbName[0]) { - createTbReq.name = strdup(pDataBlock->info.parTbName); - } else { - createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); - } - - // save schema len - int32_t code; - int32_t schemaLen; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqSz, &schemaLen); - - // save schema str - void* schemaStr = taosMemoryMalloc(schemaLen); - if (schemaStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - return NULL; - } - taosArrayPush(schemaReqs, &schemaStr); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, schemaStr, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - if (code < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - tEncoderClear(&encoder); - return NULL; - } - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - } - } - taosArrayDestroy(tagArray); - - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - continue; - } - int32_t rows = pDataBlock->info.rows; - /*int32_t rowSize = pDataBlock->info.rowSize;*/ - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - - int32_t schemaLen = 0; - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - } - cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; - } - - // assign data - ret = rpcMallocCont(cap); - ret->header.vgId = pVnode->config.vgId; - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); - - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - if (pDataBlock->info.type == STREAM_DELETE_RESULT) { - pDeleteReq->suid = suid; - pDeleteReq->deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, pDeleteReq); - continue; - } - - blkHead->numOfRows = htonl(pDataBlock->info.rows); - blkHead->sversion = htonl(pTSchema->version); - blkHead->suid = htobe64(suid); - // uid is assigned by vnode - blkHead->uid = 0; - - int32_t rows = pDataBlock->info.rows; - - tqDebug("tq sink, convert block1 %d, rows: %d", i, rows); - - int32_t dataLen = 0; - int32_t schemaLen = 0; - void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - if (createTb) { - schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); - void* schemaStr = taosArrayGetP(schemaReqs, i); - memcpy(blkSchema, schemaStr, schemaLen); - } - blkHead->schemaLen = htonl(schemaLen); - - STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - for (int32_t j = 0; j < rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); - - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); - if (colDataIsNull_s(pColData, j)) { - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - tdSRowEnd(&rb); - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; - } - blkHead->dataLen = htonl(dataLen); - - ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + schemaLen + dataLen); - } - - ret->length = htonl(ret->length); - - taosArrayDestroyP(schemaReqs, taosMemoryFree); - taosArrayDestroy(schemaReqSz); - - return ret; -} - void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { const SArray* pBlocks = (const SArray*)data; SVnode* pVnode = (SVnode*)vnode; @@ -334,8 +105,8 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d int32_t code; tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); if (code < 0) { - // - ASSERT(0); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return; } SEncoder encoder; void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); @@ -359,7 +130,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d char* ctbName = NULL; // set child table name if (pDataBlock->info.parTbName[0]) { - ctbName = strdup(pDataBlock->info.parTbName); + ctbName = taosStrdup(pDataBlock->info.parTbName); } else { ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); } @@ -384,7 +155,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d // set super table name SName name = {0}; tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - createTbReq.ctb.stbName = strdup((char*)tNameGetTableName(&name)); // strdup(stbFullName); + createTbReq.ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName); createTbReq.name = ctbName; ctbName = NULL; @@ -493,7 +264,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d blkHead->uid = 0; blkHead->schemaLen = 0; - tqDebug("tq sink, convert block2 %d, rows: %d", i, rows); + tqDebug("tq sink pipe1, convert block2 %d, rows: %d", i, rows); int32_t dataLen = 0; void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); @@ -522,7 +293,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d } else { void* colData = colDataGetData(pColData, j); if (k == 0) { - tqDebug("tq sink, row %d ts %" PRId64, j, *(int64_t*)colData); + tqDebug("tq sink pipe1, row %d ts %" PRId64, j, *(int64_t*)colData); } tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, colData, true, pColumn->offset, k); } @@ -551,60 +322,406 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d taosArrayDestroy(tagArray); } -#if 0 -void tqSinkToTableMerge(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { - const SArray* pRes = (const SArray*)data; - SVnode* pVnode = (SVnode*)vnode; - SBatchDeleteReq deleteReq = {0}; +static int32_t encodeCreateChildTableForRPC(SVCreateTbBatchReq* pReqs, int32_t vgId, void** pBuf, int32_t* contLen) { + int32_t ret = 0; - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, (int32_t)pRes->size); - - ASSERT(pTask->tbSink.pTSchema); - deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); - SSubmitReq* submitReq = tqBlockToSubmit(pVnode, pRes, pTask->tbSink.pTSchema, pTask->tbSink.pSchemaWrapper, true, - pTask->tbSink.stbUid, pTask->tbSink.stbFullName, &deleteReq); + tEncodeSize(tEncodeSVCreateTbBatchReq, pReqs, *contLen, ret); + if (ret < 0) { + ret = -1; + goto end; + } + *contLen += sizeof(SMsgHead); + *pBuf = rpcMallocCont(*contLen); + if (NULL == *pBuf) { + ret = -1; + goto end; + } + ((SMsgHead*)(*pBuf))->vgId = vgId; + ((SMsgHead*)(*pBuf))->contLen = htonl(*contLen); + SEncoder coder = {0}; + tEncoderInit(&coder, POINTER_SHIFT(*pBuf, sizeof(SMsgHead)), (*contLen) - sizeof(SMsgHead)); + if (tEncodeSVCreateTbBatchReq(&coder, pReqs) < 0) { + rpcFreeCont(*pBuf); + *pBuf = NULL; + *contLen = 0; + tEncoderClear(&coder); + ret = -1; + goto end; + } + tEncoderClear(&coder); - tqDebug("vgId:%d, task %d convert blocks over, put into write-queue", TD_VID(pVnode), pTask->taskId); +end: + return ret; +} - if (taosArrayGetSize(deleteReq.deleteReqs) != 0) { - int32_t code; - int32_t len; - tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); - if (code < 0) { - // - ASSERT(0); - } - SEncoder encoder; - void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); - void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); - tEncoderInit(&encoder, abuf, len); - tEncodeSBatchDeleteReq(&encoder, &deleteReq); - tEncoderClear(&encoder); - - ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; - - SRpcMsg msg = { - .msgType = TDMT_VND_BATCH_DEL, - .pCont = serializedDeleteReq, - .contLen = len + sizeof(SMsgHead), - }; - if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - rpcFreeCont(serializedDeleteReq); - tqDebug("failed to put into write-queue since %s", terrstr()); - } - } - taosArrayDestroy(deleteReq.deleteReqs); +int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) { + void* buf = NULL; + int32_t tlen = 0; + encodeCreateChildTableForRPC(pReqs, TD_VID(pVnode), &buf, &tlen); - /*tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema);*/ - // build write msg SRpcMsg msg = { - .msgType = TDMT_VND_SUBMIT, - .pCont = submitReq, - .contLen = ntohl(submitReq->length), + .msgType = TDMT_VND_CREATE_TABLE, + .pCont = buf, + .contLen = tlen, }; if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { - tqDebug("failed to put into write-queue since %s", terrstr()); + tqError("failed to put into write-queue since %s", terrstr()); } + + return TSDB_CODE_SUCCESS; + +_error: + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + return TSDB_CODE_OUT_OF_MEMORY; +} + +void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* data) { + const SArray* pBlocks = (const SArray*)data; + SVnode* pVnode = (SVnode*)vnode; + int64_t suid = pTask->tbSink.stbUid; + char* stbFullName = pTask->tbSink.stbFullName; + STSchema* pTSchema = pTask->tbSink.pTSchema; + /*SSchemaWrapper* pSchemaWrapper = pTask->tbSink.pSchemaWrapper;*/ + + int32_t blockSz = taosArrayGetSize(pBlocks); + + tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + + void* pBuf = NULL; + SArray* tagArray = NULL; + SArray* pVals = NULL; + SArray* crTblArray = NULL; + + for (int32_t i = 0; i < blockSz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + SBatchDeleteReq deleteReq = {0}; + deleteReq.deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq)); + deleteReq.suid = suid; + tqBuildDeleteReq(pVnode, stbFullName, pDataBlock, &deleteReq); + if (taosArrayGetSize(deleteReq.deleteReqs) == 0) { + taosArrayDestroy(deleteReq.deleteReqs); + continue; + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSBatchDeleteReq, &deleteReq, len, code); + if (code < 0) { + // + ASSERT(0); + } + SEncoder encoder; + void* serializedDeleteReq = rpcMallocCont(len + sizeof(SMsgHead)); + void* abuf = POINTER_SHIFT(serializedDeleteReq, sizeof(SMsgHead)); + tEncoderInit(&encoder, abuf, len); + tEncodeSBatchDeleteReq(&encoder, &deleteReq); + tEncoderClear(&encoder); + taosArrayDestroy(deleteReq.deleteReqs); + + ((SMsgHead*)serializedDeleteReq)->vgId = pVnode->config.vgId; + + SRpcMsg msg = { + .msgType = TDMT_VND_BATCH_DEL, + .pCont = serializedDeleteReq, + .contLen = len + sizeof(SMsgHead), + }; + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put delete req into write-queue since %s", terrstr()); + } + } else if (pDataBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + SVCreateTbBatchReq reqs = {0}; + crTblArray = reqs.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); + if (NULL == reqs.pArray) { + goto _end; + } + for (int32_t rowId = 0; rowId < rows; rowId++) { + SVCreateTbReq createTbReq = {0}; + SVCreateTbReq* pCreateTbReq = &createTbReq; + if (!pCreateTbReq) { + goto _end; + } + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName); + + // set tag content + int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); + if (size == 2) { + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + } else { + tagArray = taosArrayInit(size - 1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { + SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); + STagVal tagVal = { + .cid = pTSchema->numOfCols + step, + .type = pTagData->info.type, + }; + void* pData = colDataGetData(pTagData, rowId); + if (colDataIsNull_s(pTagData, rowId)) { + continue; + } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { + tagVal.nData = varDataLen(pData); + tagVal.pData = varDataVal(pData); + } else { + memcpy(&tagVal.i64, pData, pTagData->info.bytes); + } + taosArrayPush(tagArray, &tagVal); + } + } + pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + + + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set table name + if (!pDataBlock->info.parTbName[0]) { + SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); + pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); + } else { + pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName); + } + taosArrayPush(reqs.pArray, pCreateTbReq); + } + reqs.nReqs = taosArrayGetSize(reqs.pArray); + if (tqPutReqToQueue(pVnode, &reqs) != TSDB_CODE_SUCCESS) { + goto _end; + } + tagArray = taosArrayDestroy(tagArray); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + crTblArray = NULL; + } else { + SSubmitTbData tbData = {0}; + tqDebug("tq sink pipe2, convert block1 %d, rows: %d", i, rows); + + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + + tbData.suid = suid; + tbData.uid = 0; // uid is assigned by vnode + tbData.sver = pTSchema->version; + + char* ctbName = NULL; + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), pDataBlock->info.parTbName); + if (pDataBlock->info.parTbName[0]) { + ctbName = taosStrdup(pDataBlock->info.parTbName); + } else { + ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId); + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, pVnode->pMeta, 0); + if (metaGetTableEntryByName(&mr, ctbName) < 0) { + metaReaderClear(&mr); + tqDebug("vgId:%d, stream write into %s, table auto created", TD_VID(pVnode), ctbName); + + SVCreateTbReq* pCreateTbReq = NULL; + + if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) { + goto _end; + }; + + // set const + pCreateTbReq->flags = 0; + pCreateTbReq->type = TSDB_CHILD_TABLE; + pCreateTbReq->ctb.suid = suid; + + // set super table name + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName); + + // set tag content + tagArray = taosArrayInit(1, sizeof(STagVal)); + if (!tagArray) { + goto _end; + } + STagVal tagVal = { + .cid = pTSchema->numOfCols + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.id.groupId, + }; + taosArrayPush(tagArray, &tagVal); + pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray); + + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); + tagArray = taosArrayDestroy(tagArray); + if (pTag == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + pCreateTbReq->ctb.pTag = (uint8_t*)pTag; + + // set tag name + SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN); + char tagNameStr[TSDB_COL_NAME_LEN] = {0}; + strcpy(tagNameStr, "group_id"); + taosArrayPush(tagName, tagNameStr); + pCreateTbReq->ctb.tagName = tagName; + + // set table name + pCreateTbReq->name = ctbName; + ctbName = NULL; + + tbData.pCreateTbReq = pCreateTbReq; + tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; + } else { + if (mr.me.type != TSDB_CHILD_TABLE) { + tqError("vgId:%d, failed to write into %s, since table type incorrect, type %d", TD_VID(pVnode), ctbName, + mr.me.type); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + continue; + } + + if (mr.me.ctbEntry.suid != suid) { + tqError("vgId:%d, failed to write into %s, since suid mismatch, expect suid: %" PRId64 + ", actual suid %" PRId64 "", + TD_VID(pVnode), ctbName, suid, mr.me.ctbEntry.suid); + metaReaderClear(&mr); + taosMemoryFree(ctbName); + } + + tbData.uid = mr.me.uid; + metaReaderClear(&mr); + taosMemoryFreeClear(ctbName); + } + + // rows + if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } + + for (int32_t j = 0; j < rows; j++) { + taosArrayClear(pVals); + int32_t dataIndex = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pCol = &pTSchema->columns[k]; + if (k == 0) { + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + void* colData = colDataGetData(pColData, j); + tqDebug("tq sink pipe2, row %d, col %d ts %" PRId64, j, k, *(int64_t*)colData); + } + if (IS_SET_NULL(pCol)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, dataIndex); + if (colDataIsNull_s(pColData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + dataIndex++; + } else { + void* colData = colDataGetData(pColData, j); + if (IS_STR_DATA_TYPE(pCol->type)) { + SValue sv = + (SValue){.nData = varDataLen(colData), .pData = varDataVal(colData)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } else { + SValue sv; + memcpy(&sv.val, colData, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + dataIndex++; + } + } + } + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + ASSERT(pRow); + taosArrayPush(tbData.aRowP, &pRow); + } + + SSubmitReq2 submitReq = {0}; + if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + goto _end; + } + + taosArrayPush(submitReq.aSubmitTbData, &tbData); + + // encode + int32_t len; + int32_t code; + tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code); + SEncoder encoder; + len += sizeof(SSubmitReq2Msg); + pBuf = rpcMallocCont(len); + if (NULL == pBuf) { + goto _end; + } + ((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode); + ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); + ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); + if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to encode submit req since %s", terrstr()); + tEncoderClear(&encoder); + rpcFreeCont(pBuf); + continue; + } + tEncoderClear(&encoder); + tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + + SRpcMsg msg = { + .msgType = TDMT_VND_SUBMIT, + .pCont = pBuf, + .contLen = len, + }; + + if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) { + tqDebug("failed to put into write-queue since %s", terrstr()); + } + } + } +_end: + taosArrayDestroy(tagArray); + taosArrayDestroy(pVals); + taosArrayDestroyEx(crTblArray, (FDelete)tdDestroySVCreateTbReq); + // TODO: change } -#endif diff --git a/source/dnode/vnode/src/tq/tqSnapshot.c b/source/dnode/vnode/src/tq/tqSnapshot.c index 129a2dd8b309a78a1ef31bde0ab5367167999021..5c0649c10985b05dc1f5677fa9bfe1812dd07808 100644 --- a/source/dnode/vnode/src/tq/tqSnapshot.c +++ b/source/dnode/vnode/src/tq/tqSnapshot.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index b1f00bdf7446789fa3c572ff366bd20319db10ec..ab7093a701e0a5207b5f64072350ba5bc2bae703 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -168,7 +166,6 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { if (rollback) { tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); - ASSERT(0); } else { code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index 305378bc932f0484c71a9d1c91935a580189488e..ab7093a701e0a5207b5f64072350ba5bc2bae703 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -100,8 +100,6 @@ int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { } } - ASSERT(pVal && vLen); - *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen); if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -146,7 +144,7 @@ int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** p pWriter->sver = sver; pWriter->ever = ever; - if (tdbBegin(pTq->pMetaStore, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { + if (tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0) < 0) { code = -1; taosMemoryFree(pWriter); goto _err; @@ -167,12 +165,11 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) { STQ* pTq = pWriter->pTq; if (rollback) { - tdbAbort(pWriter->pTq->pMetaStore, pWriter->txn); - ASSERT(0); + tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn); } else { - code = tdbCommit(pWriter->pTq->pMetaStore, pWriter->txn); + code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; - code = tdbPostCommit(pWriter->pTq->pMetaStore, pWriter->txn); + code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn); if (code) goto _err; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 0b671e2bf5c54318b7447c7d02f2126dad3ab6ca..79e0ccc3e338a2c58815db448d70ad4f4e5e63bd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -127,12 +127,10 @@ int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -160,12 +158,10 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { } } + taosLRUCacheRelease(pCache, h, invalidate); if (invalidate) { - taosLRUCacheRelease(pCache, h, true); - } else { - taosLRUCacheRelease(pCache, h, false); + taosLRUCacheErase(pCache, key, keyLen); } - // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } return code; @@ -226,7 +222,7 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { return code; } -int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, STSRow *row, bool dup) { +int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, TSDBROW *row, bool dup) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -237,13 +233,51 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + if (nCol != pTSchema->numOfCols) { invalidate = true; goto _invalidate; @@ -263,7 +297,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (colVal.cid != tColVal->cid) { invalidate = true; @@ -309,7 +343,7 @@ int32_t tsdbCacheInsertLastrow(SLRUCache *pCache, STsdb *pTsdb, tb_uid_t uid, ST return code; } -int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb) { +int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, TSDBROW *row, STsdb *pTsdb) { int32_t code = 0; STSRow *cacheRow = NULL; char key[32] = {0}; @@ -320,13 +354,51 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); if (h) { STSchema *pTSchema = metaGetTbTSchema(pTsdb->pVnode->pMeta, uid, -1, 1); - TSKEY keyTs = row->ts; + TSKEY keyTs = TSDBROW_TS(row); bool invalidate = false; SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h); int16_t nCol = taosArrayGetSize(pLast); int16_t iCol = 0; + if (nCol <= 0) { + nCol = pTSchema->numOfCols; + + STColumn *pTColumn = &pTSchema->columns[0]; + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); + if (taosArrayPush(pLast, &(SLastCol){.ts = keyTs, .colVal = tColVal}) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + + for (iCol = 1; iCol < nCol; ++iCol) { + SColVal colVal = {0}; + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); + + SLastCol lastCol = {.ts = keyTs, .colVal = colVal}; + if (IS_VAR_DATA_TYPE(colVal.type) && colVal.value.nData > 0) { + SLastCol *pLastCol = (SLastCol *)taosArrayGet(pLast, iCol); + taosMemoryFree(pLastCol->colVal.value.pData); + + lastCol.colVal.value.pData = taosMemoryMalloc(colVal.value.nData); + if (lastCol.colVal.value.pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + memcpy(lastCol.colVal.value.pData, colVal.value.pData, colVal.value.nData); + } + + if (taosArrayPush(pLast, &lastCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _invalidate; + } + } + + goto _invalidate; + } + if (nCol != pTSchema->numOfCols) { invalidate = true; goto _invalidate; @@ -346,7 +418,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SColVal *tColVal = &tTsVal1->colVal; SColVal colVal = {0}; - tTSRowGetVal(row, pTSchema, iCol, &colVal); + tsdbRowGetColVal(row, pTSchema, iCol, &colVal); if (colVal.cid != tColVal->cid) { invalidate = true; @@ -678,7 +750,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } else { // tBlockDataDestroy(&state->blockData, 1); if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -707,12 +779,11 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { code = tsdbReadBlockIdx(*state->pDataFReader, state->aBlockIdx); if (code) goto _err; */ - int32_t code = - tsdbCacheGetBlockIdx((*state->pDataFReader)->pTsdb->biCache, *state->pDataFReader, &state->aBlockIdxHandle); + int32_t code = tsdbCacheGetBlockIdx(state->pTsdb->biCache, *state->pDataFReader, &state->aBlockIdxHandle); if (code != TSDB_CODE_SUCCESS || state->aBlockIdxHandle == NULL) { goto _err; } - state->aBlockIdx = (SArray *)taosLRUCacheValue((*state->pDataFReader)->pTsdb->biCache, state->aBlockIdxHandle); + state->aBlockIdx = (SArray *)taosLRUCacheValue(state->pTsdb->biCache, state->aBlockIdxHandle); /* if (state->pBlockIdx) { */ /* } */ @@ -742,7 +813,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (!state->pBlockData) { state->pBlockData = &state->blockData; - tBlockDataCreate(&state->blockData); + code = tBlockDataCreate(&state->blockData); + if (code) goto _err; } } case SFSNEXTROW_BLOCKDATA: @@ -754,7 +826,8 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { tBlockDataReset(state->pBlockData); tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); - /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ + /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); + */ tBlockDataReset(state->pBlockData); TABLEID tid = {.suid = state->suid, .uid = state->uid}; code = tBlockDataInit(state->pBlockData, &tid, state->pTSchema, NULL, 0); @@ -782,7 +855,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (state->aBlockIdx) { // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease((*state->pDataFReader)->pTsdb->biCache, state->aBlockIdxHandle); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); state->aBlockIdxHandle = NULL; state->aBlockIdx = NULL; @@ -808,13 +881,13 @@ _err: }*/ if (state->aBlockIdx) { // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease((*state->pDataFReader)->pTsdb->biCache, state->aBlockIdxHandle); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); state->aBlockIdxHandle = NULL; state->aBlockIdx = NULL; } if (state->pBlockData) { - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -837,14 +910,14 @@ int32_t clearNextRowFromFS(void *iter) { }*/ if (state->aBlockIdx) { // taosArrayDestroy(state->aBlockIdx); - tsdbBICacheRelease((*state->pDataFReader)->pTsdb->biCache, state->aBlockIdxHandle); + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); state->aBlockIdxHandle = NULL; state->aBlockIdx = NULL; } if (state->pBlockData) { // tBlockDataDestroy(&state->blockData, 1); - tBlockDataDestroy(state->pBlockData, 1); + tBlockDataDestroy(state->pBlockData); state->pBlockData = NULL; } @@ -1299,12 +1372,12 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo // build the result ts row here *dup = false; - if (taosArrayGetSize(pColArray) != nCol) { - *ppColArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppColArray = pColArray; - } + // if (taosArrayGetSize(pColArray) != nCol) { + //*ppColArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppColArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1417,12 +1490,12 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach } } while (setNoneCol); - if (taosArrayGetSize(pColArray) <= 0) { - *ppLastArray = NULL; - taosArrayDestroy(pColArray); - } else { - *ppLastArray = pColArray; - } + // if (taosArrayGetSize(pColArray) <= 0) { + //*ppLastArray = NULL; + // taosArrayDestroy(pColArray); + //} else { + *ppLastArray = pColArray; + //} nextRowIterClose(&iter); // taosMemoryFreeClear(pTSchema); @@ -1453,8 +1526,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * SArray *pArray = NULL; bool dup = false; // which is always false for now code = mergeLastRow(uid, pTsdb, &dup, &pArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pArray == NULL*/) { if (!dup && pArray) { taosArrayDestroy(pArray); } @@ -1472,13 +1545,8 @@ int32_t tsdbCacheGetLastrowH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader * if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; @@ -1502,8 +1570,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (!h) { SArray *pLastArray = NULL; code = mergeLast(uid, pTsdb, &pLastArray, pr); - // if table's empty or error, return code of -1 - if (code < 0 || pLastArray == NULL) { + // if table's empty or error, set handle NULL and return + if (code < 0 /* || pLastArray == NULL*/) { taosThreadMutexUnlock(&pTsdb->lruMutex); *handle = NULL; @@ -1517,13 +1585,8 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, SCacheRowsReader *pr, if (status != TAOS_LRU_STATUS_OK) { code = -1; } - - // taosThreadMutexUnlock(&pTsdb->lruMutex); - - // h = taosLRUCacheLookup(pCache, key, keyLen); - } // else { + } taosThreadMutexUnlock(&pTsdb->lruMutex); - //} } *handle = h; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f77997caad6a57dee3e8acb597cb88fac640f383..97b13a3c3772e2309a31e6e4d08e159032ff24bc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -148,6 +148,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; + p->pTsdb = p->pVnode->pTsdb; + p->verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; p->numOfCols = numOfCols; p->suid = suid; @@ -188,7 +190,8 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, return TSDB_CODE_OUT_OF_MEMORY; } - p->idstr = taosMemoryStrDup(idstr); + p->idstr = taosStrdup(idstr); + taosThreadMutexInit(&p->readerMutex, NULL); *pReader = p; return TSDB_CODE_SUCCESS; @@ -209,6 +212,8 @@ void* tsdbCacherowsReaderClose(void* pReader) { destroyLastBlockLoadInfo(p->pLoadInfo); taosMemoryFree((void*)p->idstr); + taosThreadMutexDestroy(&p->readerMutex); + taosMemoryFree(pReader); return NULL; } @@ -243,6 +248,27 @@ static void freeItem(void* pItem) { } } +static int32_t tsdbCacheQueryReseek(void* pQHandle) { + int32_t code = 0; + SCacheRowsReader* pReader = pQHandle; + + code = taosThreadMutexTryLock(&pReader->readerMutex); + if (code == 0) { + // pause current reader's state if not paused, save ts & version for resuming + // just wait for the big all tables' snapshot untaking for now + + code = TSDB_CODE_VND_QUERY_BUSY; + + taosThreadMutexUnlock(&pReader->readerMutex); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + return -1; + } +} + int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32_t* slotIds, SArray* pTableUidList) { if (pReader == NULL || pResBlock == NULL) { return TSDB_CODE_INVALID_PARA; @@ -285,11 +311,11 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 taosArrayPush(pLastCols, &p); } - code = tsdbTakeReadSnap(pr->pVnode->pTsdb, &pr->pReadSnap, "cache-l"); + taosThreadMutexLock(&pr->readerMutex); + code = tsdbTakeReadSnap((STsdbReader*)pr, tsdbCacheQueryReseek, &pr->pReadSnap); if (code != TSDB_CODE_SUCCESS) { goto _end; } - pr->pDataFReader = NULL; pr->pDataFReaderLast = NULL; @@ -306,6 +332,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } { for (int32_t k = 0; k < pr->numOfCols; ++k) { @@ -375,6 +405,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 if (h == NULL) { continue; } + if (taosArrayGetSize(pRow) <= 0) { + tsdbCacheRelease(lruCache, h); + continue; + } saveOneRow(pRow, pResBlock, pr, slotIds, pRes, pr->idstr); // TODO reset the pRes @@ -395,11 +429,14 @@ _end: tsdbDataFReaderClose(&pr->pDataFReaderLast); tsdbDataFReaderClose(&pr->pDataFReader); - tsdbUntakeReadSnap(pr->pVnode->pTsdb, pr->pReadSnap, "cache-l"); resetLastBlockLoadInfo(pr->pLoadInfo); + tsdbUntakeReadSnap((STsdbReader*)pr, pr->pReadSnap, true); + taosThreadMutexUnlock(&pr->readerMutex); - for (int32_t j = 0; j < pr->numOfCols; ++j) { - taosMemoryFree(pRes[j]); + if (pRes != NULL) { + for (int32_t j = 0; j < pr->numOfCols; ++j) { + taosMemoryFree(pRes[j]); + } } taosMemoryFree(pRes); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8ec59ea95949a4bccda38c7cc2fda25e2f769fb4..d15f848cfdcde11c946a943d96b0397933324b14 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -175,7 +175,7 @@ int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { pTsdb->imem = NULL; taosThreadRwlockUnlock(&pTsdb->rwLock); - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); goto _exit; } @@ -913,24 +913,24 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { // reader taosArrayDestroy(pCommitter->dReader.aBlockIdx); tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData, 1); + tBlockDataDestroy(&pCommitter->dReader.bData); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { SDataIter *pIter = &pCommitter->aDataIter[iStt]; taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData, 1); + tBlockDataDestroy(&pIter->bData); } // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData, 1); + tBlockDataDestroy(&pCommitter->dWriter.bData); #if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); #else - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tBlockDataDestroy(&pCommitter->dWriter.bDatal); #endif tDestroyTSchema(pCommitter->skmTable.pTSchema); tDestroyTSchema(pCommitter->skmRow.pTSchema); @@ -1183,7 +1183,6 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) tBlockDataClear(pBlockData); while (pRowInfo) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1251,7 +1250,6 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) pRow = NULL; } } else if (c > 0) { - ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); @@ -1493,7 +1491,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1536,7 +1534,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { while (pRowInfo) { STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { + if (pRowInfo->row.type == TSDBROW_ROW_FMT) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; @@ -1666,7 +1664,7 @@ int32_t tsdbFinishCommit(STsdb *pTsdb) { // unlock taosThreadRwlockUnlock(&pTsdb->rwLock); if (pMemTable) { - tsdbUnrefMemTable(pMemTable); + tsdbUnrefMemTable(pMemTable, NULL, true); } _exit: diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c index fb3917be64faa058b52f1f13a86ec0034486f279..fc7df98217cfebd8c7979c2e08167330d36bddee 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCompact.c +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -15,13 +15,653 @@ #include "tsdb.h" +extern int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo); +extern int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg); +extern int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg); + typedef struct { STsdb *pTsdb; + int64_t commitID; + int8_t cmprAlg; + int32_t maxRows; + int32_t minRows; + STsdbFS fs; + + int32_t fid; + TABLEID tbid; + SSkmInfo tbSkm; + + // Tombstone + SDelFReader *pDelFReader; + SArray *aDelIdx; // SArray + SArray *aDelData; // SArray + SArray *aSkyLine; // SArray + int32_t iDelIdx; + int32_t iSkyLine; + TSDBKEY *pDKey; + TSDBKEY dKey; + + // Reader + SDataFReader *pReader; + STsdbDataIter2 *iterList; // list of iterators + STsdbDataIter2 *pIter; + SRBTree rbt; + + // Writer + SDataFWriter *pWriter; + SArray *aBlockIdx; // SArray + SMapData mDataBlk; // SMapData + SArray *aSttBlk; // SArray + SBlockData bData; + SBlockData sData; } STsdbCompactor; -int32_t tsdbCompact(STsdb *pTsdb) { +static int32_t tsdbCommitCompact(STsdbCompactor *pCompactor) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *pTsdb = pCompactor->pTsdb; + + code = tsdbFSPrepareCommit(pTsdb, &pCompactor->fs); + TSDB_CHECK_CODE(code, lino, _exit); + + taosThreadRwlockWrlock(&pTsdb->rwLock); + + code = tsdbFSCommit(pTsdb); + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + TSDB_CHECK_CODE(code, lino, _exit); + } + + taosThreadRwlockUnlock(&pTsdb->rwLock); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbAbortCompact(STsdbCompactor *pCompactor) { + int32_t code = 0; + int32_t lino = 0; + + STsdb *pTsdb = pCompactor->pTsdb; + code = tsdbFSRollback(pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); + } + return code; +} + +static int32_t tsdbCompactWriteTableDataStart(STsdbCompactor *pCompactor, TABLEID *pId) { + int32_t code = 0; + int32_t lino = 0; + + pCompactor->tbid = *pId; + + // tombstone + for (;;) { + if (pCompactor->iDelIdx >= taosArrayGetSize(pCompactor->aDelIdx)) { + pCompactor->pDKey = NULL; + break; + } + + SDelIdx *pDelIdx = (SDelIdx *)taosArrayGet(pCompactor->aDelIdx, pCompactor->iDelIdx); + int32_t c = tTABLEIDCmprFn(pDelIdx, &pCompactor->tbid); + if (c < 0) { + pCompactor->iDelIdx++; + } else if (c == 0) { + pCompactor->iDelIdx++; + + code = tsdbReadDelData(pCompactor->pDelFReader, pDelIdx, pCompactor->aDelData); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbBuildDeleteSkyline(pCompactor->aDelData, 0, taosArrayGetSize(pCompactor->aDelData) - 1, + pCompactor->aSkyLine); + TSDB_CHECK_CODE(code, lino, _exit); + + pCompactor->iSkyLine = 0; + if (pCompactor->iSkyLine < taosArrayGetSize(pCompactor->aSkyLine)) { + TSDBKEY *pKey = (TSDBKEY *)taosArrayGet(pCompactor->aSkyLine, pCompactor->iSkyLine); + + pCompactor->dKey.version = 0; + pCompactor->dKey.ts = pKey->ts; + pCompactor->pDKey = &pCompactor->dKey; + } else { + pCompactor->pDKey = NULL; + } + break; + } else { + pCompactor->pDKey = NULL; + break; + } + } + + // writer + code = tsdbUpdateTableSchema(pCompactor->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pCompactor->tbSkm); + TSDB_CHECK_CODE(code, lino, _exit); + + tMapDataReset(&pCompactor->mDataBlk); + + code = tBlockDataInit(&pCompactor->bData, pId, pCompactor->tbSkm.pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + + if (!TABLE_SAME_SCHEMA(pCompactor->sData.suid, pCompactor->sData.uid, pId->suid, pId->uid)) { + if (pCompactor->sData.nRow > 0) { + code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } + + TABLEID tbid = {.suid = pId->suid, .uid = pId->suid ? 0 : pId->uid}; + code = tBlockDataInit(&pCompactor->sData, &tbid, pCompactor->tbSkm.pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pId->suid, + pId->uid); + } + return code; +} + +static int32_t tsdbCompactWriteTableDataEnd(STsdbCompactor *pCompactor) { + int32_t code = 0; + int32_t lino = 0; + + if (pCompactor->bData.nRow > 0) { + if (pCompactor->bData.nRow < pCompactor->minRows) { + for (int32_t iRow = 0; iRow < pCompactor->bData.nRow; iRow++) { + code = tBlockDataAppendRow(&pCompactor->sData, &tsdbRowFromBlockData(&pCompactor->bData, iRow), NULL, + pCompactor->tbid.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCompactor->sData.nRow >= pCompactor->maxRows) { + code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + tBlockDataClear(&pCompactor->bData); + } else { + code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (pCompactor->mDataBlk.nItem > 0) { + SBlockIdx *pBlockIdx = (SBlockIdx *)taosArrayReserve(pCompactor->aBlockIdx, 1); + if (pBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + pBlockIdx->suid = pCompactor->tbid.suid; + pBlockIdx->uid = pCompactor->tbid.uid; + + code = tsdbWriteDataBlk(pCompactor->pWriter, &pCompactor->mDataBlk, pBlockIdx); + TSDB_CHECK_CODE(code, lino, _exit); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, + pCompactor->tbid.suid, pCompactor->tbid.uid); + } + return code; +} + +static bool tsdbCompactRowIsDeleted(STsdbCompactor *pCompactor, TSDBROW *pRow) { + TSDBKEY tKey = TSDBROW_KEY(pRow); + TSDBKEY *aKey = (TSDBKEY *)TARRAY_DATA(pCompactor->aSkyLine); + int32_t nKey = TARRAY_SIZE(pCompactor->aSkyLine); + + if (tKey.ts > pCompactor->pDKey->ts) { + do { + pCompactor->pDKey->version = aKey[pCompactor->iSkyLine].version; + pCompactor->iSkyLine++; + if (pCompactor->iSkyLine < nKey) { + pCompactor->dKey.ts = aKey[pCompactor->iSkyLine].ts; + } else { + if (pCompactor->pDKey->version == 0) { + pCompactor->pDKey = NULL; + return false; + } else { + pCompactor->pDKey->ts = INT64_MAX; + } + } + } while (tKey.ts > pCompactor->pDKey->ts); + } + + if (tKey.ts < pCompactor->pDKey->ts) { + if (tKey.version > pCompactor->pDKey->version) { + return false; + } else { + return true; + } + } else if (tKey.ts == pCompactor->pDKey->ts) { + ASSERT(pCompactor->iSkyLine < nKey); + if (tKey.version > TMAX(pCompactor->pDKey->version, aKey[pCompactor->iSkyLine].version)) { + return false; + } else { + return true; + } + } + + return false; +} + +static int32_t tsdbCompactWriteTableData(STsdbCompactor *pCompactor, SRowInfo *pRowInfo) { + int32_t code = 0; + int32_t lino = 0; + + // start a new table data write if need + if (pRowInfo == NULL || pRowInfo->uid != pCompactor->tbid.uid) { + if (pCompactor->tbid.uid) { + code = tsdbCompactWriteTableDataEnd(pCompactor); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (pRowInfo == NULL) { + if (pCompactor->sData.nRow > 0) { + code = tsdbWriteSttBlock(pCompactor->pWriter, &pCompactor->sData, pCompactor->aSttBlk, pCompactor->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } + return code; + } + + code = tsdbCompactWriteTableDataStart(pCompactor, (TABLEID *)pRowInfo); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // check if row is deleted + if (pCompactor->pDKey && tsdbCompactRowIsDeleted(pCompactor, &pRowInfo->row)) goto _exit; + + if (tBlockDataTryUpsertRow(&pCompactor->bData, &pRowInfo->row, pRowInfo->uid) > pCompactor->maxRows) { + code = tsdbWriteDataBlock(pCompactor->pWriter, &pCompactor->bData, &pCompactor->mDataBlk, pCompactor->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataUpsertRow(&pCompactor->bData, &pRowInfo->row, NULL, pRowInfo->uid); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } else if (pRowInfo) { + tsdbTrace("vgId:%d %s done, suid:%" PRId64 " uid:%" PRId64 " ts:%" PRId64 " version:%" PRId64, + TD_VID(pCompactor->pTsdb->pVnode), __func__, pRowInfo->suid, pRowInfo->uid, TSDBROW_TS(&pRowInfo->row), + TSDBROW_VERSION(&pRowInfo->row)); + } + return code; +} + +static bool tsdbCompactTableIsDropped(STsdbCompactor *pCompactor) { + SMetaInfo info; + + if (pCompactor->pIter->rowInfo.uid == pCompactor->tbid.uid) return false; + if (metaGetInfo(pCompactor->pTsdb->pVnode->pMeta, pCompactor->pIter->rowInfo.uid, &info, NULL)) { + return true; + } + return false; +} +static int32_t tsdbCompactNextRow(STsdbCompactor *pCompactor, SRowInfo **ppRowInfo) { + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + if (pCompactor->pIter) { + code = tsdbDataIterNext2(pCompactor->pIter, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCompactor->pIter->rowInfo.suid == 0 && pCompactor->pIter->rowInfo.uid == 0) { + pCompactor->pIter = NULL; + } else { + SRBTreeNode *pNode = tRBTreeMin(&pCompactor->rbt); + if (pNode) { + int32_t c = tsdbDataIterCmprFn(&pCompactor->pIter->rbtn, pNode); + if (c > 0) { + tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); + pCompactor->pIter = NULL; + } else if (c == 0) { + ASSERT(0); + } + } + } + } + + if (pCompactor->pIter == NULL) { + SRBTreeNode *pNode = tRBTreeDropMin(&pCompactor->rbt); + if (pNode) { + pCompactor->pIter = TSDB_RBTN_TO_DATA_ITER(pNode); + } + } + + if (pCompactor->pIter) { + if (tsdbCompactTableIsDropped(pCompactor)) { + TABLEID tbid = {.suid = pCompactor->pIter->rowInfo.suid, .uid = pCompactor->pIter->rowInfo.uid}; + tRBTreeClear(&pCompactor->rbt); + for (pCompactor->pIter = pCompactor->iterList; pCompactor->pIter; pCompactor->pIter = pCompactor->pIter->next) { + code = tsdbDataIterNext2(pCompactor->pIter, + &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_TABLEID, .tbid = tbid}); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid) { + tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); + } + } + } else { + *ppRowInfo = &pCompactor->pIter->rowInfo; + break; + } + } else { + *ppRowInfo = NULL; + break; + } + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code)); + } + return code; +} + +static int32_t tsdbCompactFileSetStart(STsdbCompactor *pCompactor, SDFileSet *pSet) { + int32_t code = 0; + int32_t lino = 0; + + pCompactor->fid = pSet->fid; + pCompactor->tbid = (TABLEID){0}; + + /* tombstone */ + pCompactor->iDelIdx = 0; + + /* reader */ + code = tsdbDataFReaderOpen(&pCompactor->pReader, pCompactor->pTsdb, pSet); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbOpenDataFileDataIter(pCompactor->pReader, &pCompactor->pIter); + TSDB_CHECK_CODE(code, lino, _exit); + + tRBTreeCreate(&pCompactor->rbt, tsdbDataIterCmprFn); + if (pCompactor->pIter) { + pCompactor->pIter->next = pCompactor->iterList; + pCompactor->iterList = pCompactor->pIter; + + code = tsdbDataIterNext2(pCompactor->pIter, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid); + tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); + } + + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + code = tsdbOpenSttFileDataIter(pCompactor->pReader, iStt, &pCompactor->pIter); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCompactor->pIter) { + pCompactor->pIter->next = pCompactor->iterList; + pCompactor->iterList = pCompactor->pIter; + + code = tsdbDataIterNext2(pCompactor->pIter, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + + ASSERT(pCompactor->pIter->rowInfo.suid || pCompactor->pIter->rowInfo.uid); + tRBTreePut(&pCompactor->rbt, &pCompactor->pIter->rbtn); + } + } + pCompactor->pIter = NULL; + + /* writer */ + code = tsdbDataFWriterOpen(&pCompactor->pWriter, pCompactor->pTsdb, + &(SDFileSet){.fid = pCompactor->fid, + .diskId = pSet->diskId, + .pHeadF = &(SHeadFile){.commitID = pCompactor->commitID}, + .pDataF = &(SDataFile){.commitID = pCompactor->commitID}, + .pSmaF = &(SSmaFile){.commitID = pCompactor->commitID}, + .nSttF = 1, + .aSttF = {&(SSttFile){.commitID = pCompactor->commitID}}}); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCompactor->aBlockIdx) { + taosArrayClear(pCompactor->aBlockIdx); + } else if ((pCompactor->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + tMapDataReset(&pCompactor->mDataBlk); + + if (pCompactor->aSttBlk) { + taosArrayClear(pCompactor->aSttBlk); + } else if ((pCompactor->aSttBlk = taosArrayInit(0, sizeof(SSttBlk))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + tBlockDataReset(&pCompactor->bData); + tBlockDataReset(&pCompactor->sData); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code), pCompactor->fid); + } else { + tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid); + } + return code; +} + +static int32_t tsdbCompactFileSetEnd(STsdbCompactor *pCompactor) { + int32_t code = 0; + int32_t lino = 0; + + ASSERT(pCompactor->bData.nRow == 0); + ASSERT(pCompactor->sData.nRow == 0); + + /* update files */ + code = tsdbWriteSttBlk(pCompactor->pWriter, pCompactor->aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbWriteBlockIdx(pCompactor->pWriter, pCompactor->aBlockIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbUpdateDFileSetHeader(pCompactor->pWriter); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbFSUpsertFSet(&pCompactor->fs, &pCompactor->pWriter->wSet); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFWriterClose(&pCompactor->pWriter, 1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDataFReaderClose(&pCompactor->pReader); + TSDB_CHECK_CODE(code, lino, _exit); + + /* do clear */ + while ((pCompactor->pIter = pCompactor->iterList) != NULL) { + pCompactor->iterList = pCompactor->pIter->next; + tsdbCloseDataIter2(pCompactor->pIter); + } + + tBlockDataReset(&pCompactor->bData); + tBlockDataReset(&pCompactor->sData); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code), pCompactor->fid); + } else { + tsdbInfo("vgId:%d %s done, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->fid); + } + return code; +} + +static int32_t tsdbCompactFileSet(STsdbCompactor *pCompactor, SDFileSet *pSet) { int32_t code = 0; - // TODO + int32_t lino = 0; + + // start compact + code = tsdbCompactFileSetStart(pCompactor, pSet); + TSDB_CHECK_CODE(code, lino, _exit); + + // do compact, end with a NULL row + SRowInfo *pRowInfo; + do { + code = tsdbCompactNextRow(pCompactor, &pRowInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbCompactWriteTableData(pCompactor, pRowInfo); + TSDB_CHECK_CODE(code, lino, _exit); + } while (pRowInfo); + + // end compact + code = tsdbCompactFileSetEnd(pCompactor); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, fid:%d", TD_VID(pCompactor->pTsdb->pVnode), __func__, lino, + tstrerror(code), pCompactor->fid); + if (pCompactor->pWriter) tsdbDataFWriterClose(&pCompactor->pWriter, 0); + while ((pCompactor->pIter = pCompactor->iterList)) { + pCompactor->iterList = pCompactor->pIter->next; + tsdbCloseDataIter2(pCompactor->pIter); + } + if (pCompactor->pReader) tsdbDataFReaderClose(&pCompactor->pReader); + } + return code; +} + +static void tsdbEndCompact(STsdbCompactor *pCompactor) { + // writer + tBlockDataDestroy(&pCompactor->sData); + tBlockDataDestroy(&pCompactor->bData); + taosArrayDestroy(pCompactor->aSttBlk); + tMapDataClear(&pCompactor->mDataBlk); + taosArrayDestroy(pCompactor->aBlockIdx); + + // reader + + // tombstone + taosArrayDestroy(pCompactor->aSkyLine); + taosArrayDestroy(pCompactor->aDelData); + taosArrayDestroy(pCompactor->aDelIdx); + + // others + tDestroyTSchema(pCompactor->tbSkm.pTSchema); + tsdbFSDestroy(&pCompactor->fs); + + tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pCompactor->pTsdb->pVnode), __func__, pCompactor->commitID); +} + +static int32_t tsdbBeginCompact(STsdb *pTsdb, SCompactInfo *pInfo, STsdbCompactor *pCompactor) { + int32_t code = 0; + int32_t lino = 0; + + pCompactor->pTsdb = pTsdb; + pCompactor->commitID = pInfo->commitID; + pCompactor->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pCompactor->maxRows = pTsdb->pVnode->config.tsdbCfg.maxRows; + pCompactor->minRows = pTsdb->pVnode->config.tsdbCfg.minRows; + pCompactor->fid = INT32_MIN; + + code = tsdbFSCopy(pTsdb, &pCompactor->fs); + TSDB_CHECK_CODE(code, lino, _exit); + + /* tombstone */ + if (pCompactor->fs.pDelFile) { + code = tsdbDelFReaderOpen(&pCompactor->pDelFReader, pCompactor->fs.pDelFile, pTsdb); + TSDB_CHECK_CODE(code, lino, _exit); + + if ((pCompactor->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if ((pCompactor->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if ((pCompactor->aSkyLine = taosArrayInit(0, sizeof(TSDBKEY))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadDelIdx(pCompactor->pDelFReader, pCompactor->aDelIdx); + TSDB_CHECK_CODE(code, lino, _exit); + } + + /* reader */ + + /* writer */ + code = tBlockDataCreate(&pCompactor->bData); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tBlockDataCreate(&pCompactor->sData); + TSDB_CHECK_CODE(code, lino, _exit); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, lino, + tstrerror(code), pCompactor->commitID); + tBlockDataDestroy(&pCompactor->sData); + tBlockDataDestroy(&pCompactor->bData); + if (pCompactor->fs.pDelFile) { + taosArrayDestroy(pCompactor->aSkyLine); + taosArrayDestroy(pCompactor->aDelData); + taosArrayDestroy(pCompactor->aDelIdx); + if (pCompactor->pDelFReader) tsdbDelFReaderClose(&pCompactor->pDelFReader); + } + tsdbFSDestroy(&pCompactor->fs); + } else { + tsdbInfo("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pTsdb->pVnode), __func__, pCompactor->commitID); + } + return code; +} + +int32_t tsdbCompact(STsdb *pTsdb, SCompactInfo *pInfo) { + int32_t code = 0; + + STsdbCompactor *pCompactor = &(STsdbCompactor){0}; + + if ((code = tsdbBeginCompact(pTsdb, pInfo, pCompactor))) return code; + + for (;;) { + SDFileSet *pSet = (SDFileSet *)taosArraySearch(pCompactor->fs.aDFileSet, &(SDFileSet){.fid = pCompactor->fid}, + tDFileSetCmprFn, TD_GT); + if (pSet == NULL) { + pCompactor->fid = INT32_MAX; + break; + } + + if ((code = tsdbCompactFileSet(pCompactor, pSet))) goto _exit; + } + + if ((code = tsdbFSUpsertDelFile(&pCompactor->fs, NULL))) goto _exit; + +_exit: + if (code) { + tsdbAbortCompact(pCompactor); + } else { + tsdbCommitCompact(pCompactor); + } + tsdbEndCompact(pCompactor); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c new file mode 100644 index 0000000000000000000000000000000000000000..3299a2f497a825a546c9e613cf74ffde1c859ca3 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +// STsdbDataIter2 +/* open */ +int32_t tsdbOpenDataFileDataIter(SDataFReader* pReader, STsdbDataIter2** ppIter) { + int32_t code = 0; + int32_t lino = 0; + + // create handle + STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + pIter->type = TSDB_DATA_FILE_DATA_ITER; + pIter->dIter.pReader = pReader; + if ((pIter->dIter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataCreate(&pIter->dIter.bData); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->dIter.iBlockIdx = 0; + pIter->dIter.iDataBlk = 0; + pIter->dIter.iRow = 0; + + // read data + code = tsdbReadBlockIdx(pReader, pIter->dIter.aBlockIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(pIter->dIter.aBlockIdx) == 0) goto _clear; + +_exit: + if (code) { + if (pIter) { + _clear: + tBlockDataDestroy(&pIter->dIter.bData); + taosArrayDestroy(pIter->dIter.aBlockIdx); + taosMemoryFree(pIter); + pIter = NULL; + } + } + *ppIter = pIter; + return code; +} + +int32_t tsdbOpenSttFileDataIter(SDataFReader* pReader, int32_t iStt, STsdbDataIter2** ppIter) { + int32_t code = 0; + int32_t lino = 0; + + // create handle + STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + pIter->type = TSDB_STT_FILE_DATA_ITER; + pIter->sIter.pReader = pReader; + pIter->sIter.iStt = iStt; + pIter->sIter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pIter->sIter.aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tBlockDataCreate(&pIter->sIter.bData); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->sIter.iSttBlk = 0; + pIter->sIter.iRow = 0; + + // read data + code = tsdbReadSttBlk(pReader, iStt, pIter->sIter.aSttBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(pIter->sIter.aSttBlk) == 0) goto _clear; + +_exit: + if (code) { + if (pIter) { + _clear: + taosArrayDestroy(pIter->sIter.aSttBlk); + tBlockDataDestroy(&pIter->sIter.bData); + taosMemoryFree(pIter); + pIter = NULL; + } + } + *ppIter = pIter; + return code; +} + +int32_t tsdbOpenTombFileDataIter(SDelFReader* pReader, STsdbDataIter2** ppIter) { + int32_t code = 0; + int32_t lino = 0; + + STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); + if (pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + pIter->type = TSDB_TOMB_FILE_DATA_ITER; + + pIter->tIter.pReader = pReader; + if ((pIter->tIter.aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + if ((pIter->tIter.aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + code = tsdbReadDelIdx(pReader, pIter->tIter.aDelIdx); + TSDB_CHECK_CODE(code, lino, _exit); + + if (taosArrayGetSize(pIter->tIter.aDelIdx) == 0) goto _clear; + + pIter->tIter.iDelIdx = 0; + pIter->tIter.iDelData = 0; + +_exit: + if (code) { + if (pIter) { + _clear: + taosArrayDestroy(pIter->tIter.aDelIdx); + taosArrayDestroy(pIter->tIter.aDelData); + taosMemoryFree(pIter); + pIter = NULL; + } + } + *ppIter = pIter; + return code; +} + +/* close */ +static void tsdbCloseDataFileDataIter(STsdbDataIter2* pIter) { + tBlockDataDestroy(&pIter->dIter.bData); + tMapDataClear(&pIter->dIter.mDataBlk); + taosArrayDestroy(pIter->dIter.aBlockIdx); + taosMemoryFree(pIter); +} + +static void tsdbCloseSttFileDataIter(STsdbDataIter2* pIter) { + tBlockDataDestroy(&pIter->sIter.bData); + taosArrayDestroy(pIter->sIter.aSttBlk); + taosMemoryFree(pIter); +} + +static void tsdbCloseTombFileDataIter(STsdbDataIter2* pIter) { + taosArrayDestroy(pIter->tIter.aDelData); + taosArrayDestroy(pIter->tIter.aDelIdx); + taosMemoryFree(pIter); +} + +void tsdbCloseDataIter2(STsdbDataIter2* pIter) { + if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) { + ASSERT(0); + } else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) { + tsdbCloseDataFileDataIter(pIter); + } else if (pIter->type == TSDB_STT_FILE_DATA_ITER) { + tsdbCloseSttFileDataIter(pIter); + } else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) { + tsdbCloseTombFileDataIter(pIter); + } else { + ASSERT(0); + } +} + +/* cmpr */ +int32_t tsdbDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { + STsdbDataIter2* pIter1 = TSDB_RBTN_TO_DATA_ITER(pNode1); + STsdbDataIter2* pIter2 = TSDB_RBTN_TO_DATA_ITER(pNode2); + return tRowInfoCmprFn(&pIter1->rowInfo, &pIter2->rowInfo); +} + +/* seek */ + +/* iter next */ +static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + while (pIter->dIter.iRow < pIter->dIter.bData.nRow) { + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) { + pIter->dIter.iRow = pIter->dIter.bData.nRow; + continue; + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { + if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver || + pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) { + pIter->dIter.iRow++; + continue; + } + } + } + + ASSERT(pIter->rowInfo.suid == pIter->dIter.bData.suid); + ASSERT(pIter->rowInfo.uid == pIter->dIter.bData.uid); + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->dIter.bData, pIter->dIter.iRow); + pIter->dIter.iRow++; + goto _exit; + } + + for (;;) { + while (pIter->dIter.iDataBlk < pIter->dIter.mDataBlk.nItem) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pIter->dIter.mDataBlk, pIter->dIter.iDataBlk, &dataBlk, tGetDataBlk); + + // filter + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) { + pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem; + continue; + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { + if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) { + pIter->dIter.iDataBlk++; + continue; + } + } + } + + code = tsdbReadDataBlockEx(pIter->dIter.pReader, &dataBlk, &pIter->dIter.bData); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->dIter.iDataBlk++; + pIter->dIter.iRow = 0; + + break; + } + + if (pIter->dIter.iRow < pIter->dIter.bData.nRow) break; + + for (;;) { + if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) { + SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); + + if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) { + int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid); + if (c == 0) { + pIter->dIter.iBlockIdx++; + continue; + } else if (c < 0) { + ASSERT(0); + } + } + + code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->rowInfo.suid = pBlockIdx->suid; + pIter->rowInfo.uid = pBlockIdx->uid; + + pIter->dIter.iBlockIdx++; + pIter->dIter.iDataBlk = 0; + + break; + } else { + pIter->rowInfo = (SRowInfo){0}; + goto _exit; + } + } + } + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + while (pIter->sIter.iRow < pIter->sIter.bData.nRow) { + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; + if (pFilterInfo->tbid.uid == uid) { + pIter->sIter.iRow++; + continue; + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { + if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] || + pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) { + pIter->sIter.iRow++; + continue; + } + } + } + + pIter->rowInfo.suid = pIter->sIter.bData.suid; + pIter->rowInfo.uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; + pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->sIter.bData, pIter->sIter.iRow); + pIter->sIter.iRow++; + goto _exit; + } + + for (;;) { + if (pIter->sIter.iSttBlk < taosArrayGetSize(pIter->sIter.aSttBlk)) { + SSttBlk* pSttBlk = taosArrayGet(pIter->sIter.aSttBlk, pIter->sIter.iSttBlk); + + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + if (pSttBlk->suid == pFilterInfo->tbid.suid && pSttBlk->minUid == pFilterInfo->tbid.uid && + pSttBlk->maxUid == pFilterInfo->tbid.uid) { + pIter->sIter.iSttBlk++; + continue; + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { + if (pFilterInfo->sver > pSttBlk->maxVer || pFilterInfo->ever < pSttBlk->minVer) { + pIter->sIter.iSttBlk++; + continue; + } + } + } + + code = tsdbReadSttBlockEx(pIter->sIter.pReader, pIter->sIter.iStt, pSttBlk, &pIter->sIter.bData); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->sIter.iRow = 0; + pIter->sIter.iSttBlk++; + break; + } else { + pIter->rowInfo = (SRowInfo){0}; + goto _exit; + } + } + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + +static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { + int32_t code = 0; + int32_t lino = 0; + + for (;;) { + while (pIter->tIter.iDelData < taosArrayGetSize(pIter->tIter.aDelData)) { + SDelData* pDelData = taosArrayGet(pIter->tIter.aDelData, pIter->tIter.iDelData); + + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { + if (pFilterInfo->sver > pDelData->version || pFilterInfo->ever < pDelData->version) { + pIter->tIter.iDelData++; + continue; + } + } + } + + pIter->delInfo.delData = *pDelData; + pIter->tIter.iDelData++; + goto _exit; + } + + for (;;) { + if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) { + SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx); + + code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); + TSDB_CHECK_CODE(code, lino, _exit); + + pIter->delInfo.suid = pDelIdx->suid; + pIter->delInfo.uid = pDelIdx->uid; + pIter->tIter.iDelData = 0; + pIter->tIter.iDelIdx++; + break; + } else { + pIter->delInfo = (SDelInfo){0}; + goto _exit; + } + } + } + +_exit: + if (code) { + tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} + +int32_t tsdbDataIterNext2(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { + int32_t code = 0; + + if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) { + ASSERT(0); + return code; + } else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) { + return tsdbDataFileDataIterNext(pIter, pFilterInfo); + } else if (pIter->type == TSDB_STT_FILE_DATA_ITER) { + return tsdbSttFileDataIterNext(pIter, pFilterInfo); + } else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) { + return tsdbTombFileDataIterNext(pIter, pFilterInfo); + } else { + ASSERT(0); + return code; + } +} + +/* get */ + +// STsdbFSetIter +typedef struct STsdbFSetDataIter { + STsdb* pTsdb; + int32_t flags; + + /* tombstone */ + SDelFReader* pDelFReader; + SArray* aDelIdx; // SArray + SArray* aDelData; // SArray + SArray* aSkeyLine; // SArray + int32_t iDelIdx; + int32_t iSkyLine; + + /* time-series data */ + SDataFReader* pReader; + STsdbDataIter2* iterList; + STsdbDataIter2* pIter; + SRBTree rbt; +} STsdbFSetDataIter; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b46a00363817666cdff7bc7756ff204483787c36..ae9af11f5ae8226c1fceefb57b4d83167800772a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -596,7 +596,7 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; STSDBRowIter iter = {0}; - tsdbRowIterInit(&iter, pRow, pTSchema); + tsdbRowIterOpen(&iter, pRow, pTSchema); SColVal *pColVal = tsdbRowIterNext(&iter); for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 51fdc69a95cc0a935a6b6e2abd390d5927384a50..5519d43012f46dc2e24ad56083ff421d5fcff1c5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -629,7 +629,15 @@ static int32_t tsdbFSApplyChange(STsdb *pTsdb, STsdbFS *pFS) { } } } else { - ASSERT(pTsdb->fs.pDelFile == NULL); + if (pTsdb->fs.pDelFile) { + nRef = atomic_sub_fetch_32(&pTsdb->fs.pDelFile->nRef, 1); + if (nRef == 0) { + tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname); + (void)taosRemoveFile(fname); + taosMemoryFree(pTsdb->fs.pDelFile); + } + pTsdb->fs.pDelFile = NULL; + } } // aDFileSet @@ -906,14 +914,21 @@ _exit: int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { int32_t code = 0; - if (pFS->pDelFile == NULL) { - pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); + if (pDelFile) { if (pFS->pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); + if (pFS->pDelFile == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + } + *pFS->pDelFile = *pDelFile; + } else { + if (pFS->pDelFile) { + taosMemoryFree(pFS->pDelFile); + pFS->pDelFile = NULL; } } - *pFS->pDelFile = *pDelFile; _exit: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 9b3dbcd8ea0d8709b1fabe39bc55ed4278eb1585..d91475376b240812b8942227fa85e37cb5671dec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -151,7 +151,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { int64_t size = 0; int64_t n; TdFilePtr pFD; - char fname[TSDB_FILENAME_LEN]; + char fname[TSDB_FILENAME_LEN] = {0}; char hdr[TSDB_FHDR_SIZE] = {0}; // truncate diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ac27a0309610338f5da22afbc3941add7bf57c71..d34af9acae68b964879cd7ba24491f0ff878e439 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -32,8 +32,10 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags); static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData); -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp); +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows); int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { int32_t code = 0; @@ -48,6 +50,8 @@ int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) { pMemTable->pTsdb = pTsdb; pMemTable->pPool = pTsdb->pVnode->inUse; pMemTable->nRef = 1; + pMemTable->minVer = VERSION_MAX; + pMemTable->maxVer = VERSION_MIN; pMemTable->minKey = TSKEY_MAX; pMemTable->maxKey = TSKEY_MIN; pMemTable->nRow = 0; @@ -70,9 +74,9 @@ _err: return code; } -void tsdbMemTableDestroy(SMemTable *pMemTable) { +void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive) { if (pMemTable) { - vnodeBufPoolUnRef(pMemTable->pPool); + vnodeBufPoolUnRef(pMemTable->pPool, proactive); taosMemoryFree(pMemTable->aBucket); taosMemoryFree(pMemTable); } @@ -99,35 +103,12 @@ STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t return pTbData; } -int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, - SSubmitBlkRsp *pRsp) { +int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { int32_t code = 0; SMemTable *pMemTable = pTsdb->mem; STbData *pTbData = NULL; - tb_uid_t suid = pMsgIter->suid; - tb_uid_t uid = pMsgIter->uid; - - SMetaInfo info; - code = metaGetInfo(pTsdb->pVnode->pMeta, uid, &info, NULL); - if (code) { - code = TSDB_CODE_TDB_TABLE_NOT_EXIST; - goto _err; - } - if (info.suid != suid) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - if (info.suid) { - metaGetInfo(pTsdb->pVnode->pMeta, info.suid, &info, NULL); - } - if (pMsgIter->sversion != info.skmVer) { - tsdbError("vgId:%d, req sver:%d, skmVer:%d suid:%" PRId64 " uid:%" PRId64, TD_VID(pTsdb->pVnode), - pMsgIter->sversion, info.skmVer, suid, uid); - code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; - goto _err; - } - - pRsp->sver = info.skmVer; + tb_uid_t suid = pSubmitTbData->suid; + tb_uid_t uid = pSubmitTbData->uid; // create/get STbData to op code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData); @@ -136,10 +117,16 @@ int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgI } // do insert impl - code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pMsgIter, pBlock, pRsp); - if (code) { - goto _err; + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + code = tsdbInsertColDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); + } else { + code = tsdbInsertRowDataToTable(pMemTable, pTbData, version, pSubmitTbData, affectedRows); } + if (code) goto _err; + + // update + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMAX(pMemTable->maxVer, version); return code; @@ -192,6 +179,8 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid } pMemTable->nDel++; + pMemTable->minVer = TMIN(pMemTable->minVer, version); + pMemTable->maxVer = TMIN(pMemTable->maxVer, version); if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && tsdbKeyCmprFn(&lastKey, &pTbData->maxKey) >= 0) { tsdbCacheDeleteLastrow(pTsdb->lruCache, pTbData->uid, eKey); @@ -201,9 +190,9 @@ int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid tsdbCacheDeleteLast(pTsdb->lruCache, pTbData->uid, eKey); } - tsdbInfo("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 - " at version %" PRId64 " since %s", - TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, version, tstrerror(code)); + tsdbTrace("vgId:%d, delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64 + " at version %" PRId64, + TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, version); return code; _err: @@ -232,7 +221,6 @@ void *tsdbTbDataIterDestroy(STbDataIter *pIter) { if (pIter) { taosMemoryFree(pIter); } - return NULL; } @@ -246,7 +234,6 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa pIter->pTbData = pTbData; pIter->backward = backward; pIter->pRow = NULL; - pIter->row.type = 0; if (pFrom == NULL) { // create from head or tail if (backward) { @@ -414,8 +401,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_GET_NODE_BACKWARD(px, iLevel); while (pn != pTbData->sl.pHead) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c <= 0) { @@ -444,8 +436,13 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) { pn = SL_GET_NODE_FORWARD(px, iLevel); while (pn != pTbData->sl.pTail) { - tKey.version = pn->version; - tKey.ts = pn->pTSRow->ts; + if (pn->flag == TSDBROW_ROW_FMT) { + tKey.version = pn->version; + tKey.ts = ((SRow *)pn->pData)->ts; + } else if (pn->flag == TSDBROW_COL_FMT) { + tKey.version = ((SBlockData *)pn->pData)->aVersion[pn->iRow]; + tKey.ts = ((SBlockData *)pn->pData)->aTSKEY[pn->iRow]; + } int32_t c = tsdbKeyCmprFn(&tKey, pKey); if (c >= 0) { @@ -472,26 +469,41 @@ static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) { return level; } -static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, int64_t version, - STSRow *pRow, int8_t forward) { +static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow, + int8_t forward) { int32_t code = 0; int8_t level; - SMemSkipListNode *pNode; + SMemSkipListNode *pNode = NULL; SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; int64_t nSize; // create node level = tsdbMemSkipListRandLevel(&pTbData->sl); nSize = SL_NODE_SIZE(level); - pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->len); + if (pRow->type == TSDBROW_ROW_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize); + } else { + ASSERT(0); + } if (pNode == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } + pNode->level = level; - pNode->version = version; - pNode->pTSRow = (STSRow *)((char *)pNode + nSize); - memcpy(pNode->pTSRow, pRow, pRow->len); + pNode->flag = pRow->type; + if (pRow->type == TSDBROW_ROW_FMT) { + pNode->version = pRow->version; + pNode->pData = (char *)pNode + nSize; + memcpy(pNode->pData, pRow->pTSRow, pRow->pTSRow->len); + } else if (pRow->type == TSDBROW_COL_FMT) { + pNode->iRow = pRow->iRow; + pNode->pData = pRow->pBlockData; + } else { + ASSERT(0); + } // set node if (forward) { @@ -536,69 +548,168 @@ _exit: return code; } -static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version, - SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) { - int32_t code = 0; - SSubmitBlkIter blkIter = {0}; - TSDBKEY key = {.version = version}; - SMemSkipListNode *pos[SL_MAX_LEVEL]; - TSDBROW row = tsdbRowFromTSRow(version, NULL); - int32_t nRow = 0; - STSRow *pLastRow = NULL; +static int32_t tsdbInsertColDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; - tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter); + SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse; + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - // backward put first data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow == NULL) return code; + ASSERT(aColData[0].cid == PRIMARYKEY_TIMESTAMP_COL_ID); + ASSERT(aColData[0].type == TSDB_DATA_TYPE_TIMESTAMP); + ASSERT(aColData[0].flag == HAS_VALUE); - key.ts = row.pTSRow->ts; - nRow++; - tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 0); - if (code) { - goto _err; + // copy and construct block data + SBlockData *pBlockData = vnodeBufPoolMalloc(pPool, sizeof(*pBlockData)); + if (pBlockData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } + pBlockData->suid = pTbData->suid; + pBlockData->uid = pTbData->uid; + pBlockData->nRow = aColData[0].nVal; + pBlockData->aUid = NULL; + pBlockData->aVersion = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aVersion == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + for (int32_t i = 0; i < pBlockData->nRow; i++) { // todo: here can be optimized + pBlockData->aVersion[i] = version; + } + + pBlockData->aTSKEY = vnodeBufPoolMalloc(pPool, aColData[0].nData); + if (pBlockData->aTSKEY == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + memcpy(pBlockData->aTSKEY, aColData[0].pData, aColData[0].nData); + + pBlockData->nColData = nColData - 1; + pBlockData->aColData = vnodeBufPoolMalloc(pPool, sizeof(SColData) * pBlockData->nColData); + if (pBlockData->aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + for (int32_t iColData = 0; iColData < pBlockData->nColData; ++iColData) { + code = tColDataCopy(&aColData[iColData + 1], &pBlockData->aColData[iColData], (xMallocFn)vnodeBufPoolMalloc, pPool); + if (code) goto _exit; + } + + // loop to add each row to the skiplist + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = tsdbRowFromBlockData(pBlockData, 0); + TSDBKEY key = {.version = version, .ts = pBlockData->aTSKEY[0]}; + TSDBROW lRow; // last row + + // first row + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0))) goto _exit; pTbData->minKey = TMIN(pTbData->minKey, key.ts); + lRow = tRow; + + // remain row + ++tRow.iRow; + if (tRow.iRow < pBlockData->nRow) { + for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { + pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); + } + + while (tRow.iRow < pBlockData->nRow) { + key.ts = pBlockData->aTSKEY[tRow.iRow]; + + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); + } + + if ((code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1))) goto _exit; + lRow = tRow; + + ++tRow.iRow; + } + } + + if (key.ts >= pTbData->maxKey) { + pTbData->maxKey = key.ts; + + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); + } + } - pLastRow = row.pTSRow; + if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); + } + + // SMemTable + pMemTable->minKey = TMIN(pMemTable->minKey, pTbData->minKey); + pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); + pMemTable->nRow += pBlockData->nRow; + + if (affectedRows) *affectedRows = pBlockData->nRow; + +_exit: + return code; +} + +static int32_t tsdbInsertRowDataToTable(SMemTable *pMemTable, STbData *pTbData, int64_t version, + SSubmitTbData *pSubmitTbData, int32_t *affectedRows) { + int32_t code = 0; + + int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); + SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + TSDBKEY key = {.version = version}; + SMemSkipListNode *pos[SL_MAX_LEVEL]; + TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .version = version}; + int32_t iRow = 0; + TSDBROW lRow; + + // backward put first data + tRow.pTSRow = aRow[iRow++]; + key.ts = tRow.pTSRow->ts; + tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD); + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 0); + if (code) goto _exit; + lRow = tRow; + + pTbData->minKey = TMIN(pTbData->minKey, key.ts); // forward put rest data - row.pTSRow = tGetSubmitBlkNext(&blkIter); - if (row.pTSRow) { + if (iRow < nRow) { for (int8_t iLevel = pos[0]->level; iLevel < pTbData->sl.maxLevel; iLevel++) { pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel); } - do { - key.ts = row.pTSRow->ts; - nRow++; + + while (iRow < nRow) { + tRow.pTSRow = aRow[iRow]; + key.ts = tRow.pTSRow->ts; + if (SL_NODE_FORWARD(pos[0], 0) != pTbData->sl.pTail) { tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS); } - code = tbDataDoPut(pMemTable, pTbData, pos, version, row.pTSRow, 1); - if (code) { - goto _err; - } - pLastRow = row.pTSRow; + code = tbDataDoPut(pMemTable, pTbData, pos, &tRow, 1); + if (code) goto _exit; + + lRow = tRow; - row.pTSRow = tGetSubmitBlkNext(&blkIter); - } while (row.pTSRow); + iRow++; + } } if (key.ts >= pTbData->maxKey) { - if (key.ts > pTbData->maxKey) { - pTbData->maxKey = key.ts; - } + pTbData->maxKey = key.ts; - if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config) && pLastRow != NULL) { - tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, pLastRow, true); + if (TSDB_CACHE_LAST_ROW(pMemTable->pTsdb->pVnode->config)) { + tsdbCacheInsertLastrow(pMemTable->pTsdb->lruCache, pMemTable->pTsdb, pTbData->uid, &lRow, true); } } if (TSDB_CACHE_LAST(pMemTable->pTsdb->pVnode->config)) { - tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, pLastRow, pMemTable->pTsdb); + tsdbCacheInsertLast(pMemTable->pTsdb->lruCache, pTbData->uid, &lRow, pMemTable->pTsdb); } // SMemTable @@ -606,27 +717,38 @@ static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, i pMemTable->maxKey = TMAX(pMemTable->maxKey, pTbData->maxKey); pMemTable->nRow += nRow; - pRsp->numOfRows = nRow; - pRsp->affectedRows = nRow; + if (affectedRows) *affectedRows = nRow; - return code; - -_err: +_exit: return code; } int32_t tsdbGetNRowsInTbData(STbData *pTbData) { return pTbData->sl.size; } -void tsdbRefMemTable(SMemTable *pMemTable) { +int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode) { + int32_t code = 0; + int32_t nRef = atomic_fetch_add_32(&pMemTable->nRef, 1); ASSERT(nRef > 0); + + vnodeBufPoolRegisterQuery(pMemTable->pPool, pQNode); + +_exit: + return code; } -void tsdbUnrefMemTable(SMemTable *pMemTable) { - int32_t nRef = atomic_sub_fetch_32(&pMemTable->nRef, 1); - if (nRef == 0) { - tsdbMemTableDestroy(pMemTable); +int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive) { + int32_t code = 0; + + if (pNode) { + vnodeBufPoolDeregisterQuery(pMemTable->pPool, pNode, proactive); + } + + if (atomic_sub_fetch_32(&pMemTable->nRef, 1) == 0) { + tsdbMemTableDestroy(pMemTable, proactive); } + + return code; } static FORCE_INLINE int32_t tbDataPCmprFn(const void *p1, const void *p2) { @@ -665,4 +787,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 91152574d222983bbb4156314f4a6adb868281fa..d4af0422d781f4560abe9ef7fe28353ff99fee0c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -91,8 +91,8 @@ void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo) { pLoadInfo[i].blockIndex[0] = -1; pLoadInfo[i].blockIndex[1] = -1; - tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); - tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[0]); + tBlockDataDestroy(&pLoadInfo[i].blockData[1]); taosArrayDestroy(pLoadInfo[i].aSttBlk); } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index f71b5b6706ccb152f4dc221d01f4a1ea6e207c7a..8901f644598ec4ca5343f4a35a7b063bf39096fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -88,7 +88,7 @@ _err: int tsdbClose(STsdb **pTsdb) { if (*pTsdb) { taosThreadRwlockWrlock(&(*pTsdb)->rwLock); - tsdbMemTableDestroy((*pTsdb)->mem); + tsdbMemTableDestroy((*pTsdb)->mem, true); (*pTsdb)->mem = NULL; taosThreadRwlockUnlock(&(*pTsdb)->rwLock); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 655177a3628e87301d5483b0f58d7ff6da993a1f..366b026f8d2af86643d218f1a74ea484844b5b84 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -79,9 +79,9 @@ typedef struct SIOCostSummary { int64_t composedBlocks; double buildComposedBlockTime; double createScanInfoList; -// double getTbFromMemTime; -// double getTbFromIMemTime; - double initDelSkylineIterTime; + // double getTbFromMemTime; + // double getTbFromIMemTime; + double initDelSkylineIterTime; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -159,6 +159,9 @@ typedef struct SBlockInfoBuf { struct STsdbReader { STsdb* pTsdb; + SVersionRange verRange; + TdThreadMutex readerMutex; + bool suspended; uint64_t suid; int16_t order; bool freeBlock; @@ -176,7 +179,6 @@ struct STsdbReader { SDataFReader* pFileReader; // the file reader SDelFReader* pDelFReader; // the del file reader SArray* pDelIdx; // del file block index; - SVersionRange verRange; SBlockInfoBuf blockInfoBuf; int32_t step; STsdbReader* innerReader[2]; @@ -192,8 +194,8 @@ static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STabl SRowMerger* pMerger, SVersionRange* pVerRange); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); -static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo); +static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, + STableBlockScanInfo* pInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -201,9 +203,9 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, - STsdbReader* pReader, STSRow** pTSRow); + STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); @@ -331,7 +333,7 @@ static int32_t uidComparFunc(const void* p1, const void* p2) { // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, - STableUidList *pUidList, int32_t numOfTables) { + STableUidList* pUidList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption SHashObj* pTableMap = @@ -477,8 +479,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb SBlockLoadSuppInfo* pInfo = &pReader->suppInfo; int32_t numOfStt = pReader->pTsdb->pVnode->config.sttTrigger; - pLReader->pInfo = - tCreateLastBlockLoadInfo(pReader->pSchema, &pInfo->colId[1], pInfo->numOfCols - 1, numOfStt); + pLReader->pInfo = tCreateLastBlockLoadInfo(pReader->pSchema, &pInfo->colId[1], pInfo->numOfCols - 1, numOfStt); if (pLReader->pInfo == NULL) { tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); return terrno; @@ -489,13 +490,15 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb return TSDB_CODE_SUCCESS; } -static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { +static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) { bool asc = ASCENDING_TRAVERSE(pIter->order); int32_t step = asc ? 1 : -1; pIter->index += step; + int32_t code = 0; if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { - return false; + *hasNext = false; + return TSDB_CODE_SUCCESS; } SIOCostSummary* pSum = &pReader->cost; @@ -515,7 +518,7 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { pReader->status.pCurrentFileset = (SDFileSet*)taosArrayGet(pIter->pFileList, pIter->index); - int32_t code = tsdbDataFReaderOpen(&pReader->pFileReader, pReader->pTsdb, pReader->status.pCurrentFileset); + code = tsdbDataFReaderOpen(&pReader->pFileReader, pReader->pTsdb, pReader->status.pCurrentFileset); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -529,24 +532,28 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { if ((asc && win.skey > pReader->window.ekey) || (!asc && win.ekey < pReader->window.skey)) { tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader, pReader->window.skey, pReader->window.ekey, pReader->idStr); - return false; + *hasNext = false; + return TSDB_CODE_SUCCESS; } if ((asc && (win.ekey < pReader->window.skey)) || ((!asc) && (win.skey > pReader->window.ekey))) { pIter->index += step; if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) { - return false; + *hasNext = false; + return TSDB_CODE_SUCCESS; } continue; } tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->window.skey, pReader->window.ekey, pReader->idStr); - return true; + *hasNext = true; + return TSDB_CODE_SUCCESS; } _err: - return false; + *hasNext = false; + return code; } static void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order) { @@ -589,6 +596,68 @@ static SSDataBlock* createResBlock(SQueryTableDataCond* pCond, int32_t capacity) return pResBlock; } +static int32_t tsdbInitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexInit(&pReader->readerMutex, NULL); + + qTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbUninitReaderLock(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexDestroy(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-take read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbTryAcquireReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexTryLock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +static int32_t tsdbReleaseReader(STsdbReader* pReader) { + int32_t code = -1; + qTrace("tsdb/read: %p, pre-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + code = taosThreadMutexUnlock(&pReader->readerMutex); + + qTrace("tsdb/read: %p, post-untake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code); + + return code; +} + +void tsdbReleaseDataBlock(STsdbReader* pReader) { + SReaderStatus* pStatus = &pReader->status; + if (!pStatus->composedDataBlock) { + tsdbReleaseReader(pReader); + } +} + static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; @@ -610,7 +679,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->order = pCond->order; pReader->capacity = capacity; pReader->pResBlock = pResBlock; - pReader->idStr = (idstr != NULL) ? strdup(idstr) : NULL; + pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); @@ -649,6 +718,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + tsdbInitReaderLock(pReader); + *ppReader = pReader; return code; @@ -659,7 +730,7 @@ _end: } static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, SArray* pIndexList) { - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); LRUHandle* handle = NULL; int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); if (code != TSDB_CODE_SUCCESS || handle == NULL) { @@ -678,11 +749,11 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, // todo binary search to the start position int64_t et1 = taosGetTimestampUs(); - SBlockIdx* pBlockIdx = NULL; + SBlockIdx* pBlockIdx = NULL; STableUidList* pList = &pReader->status.uidList; int32_t i = 0, j = 0; - while(i < num && j < numOfTables) { + while (i < num && j < numOfTables) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); if (pBlockIdx->suid != pReader->suid) { i += 1; @@ -752,7 +823,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN cleanupTableScanInfo(pReader->status.pTableMap); for (int32_t i = 0; i < numOfTables; ++i) { - SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); + SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); if (pScanInfo == NULL) { return terrno; @@ -764,12 +835,25 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN sizeInDisk += pScanInfo->mapData.nData; + int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; + STimeWindow w = pReader->window; + if (ASCENDING_TRAVERSE(pReader->order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } + + if (isEmptyQueryTimeWindow(&w)) { + continue; + } + SDataBlk block = {0}; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); // 1. time range check - if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + // if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { + if (block.minKey.ts > w.ekey || block.maxKey.ts < w.skey) { continue; } @@ -790,7 +874,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN pBlockNum->numOfBlocks += 1; } - if (pScanInfo->pBlockList != NULL && taosArrayGetSize(pScanInfo->pBlockList) > 0) { + if ((pScanInfo->pBlockList != NULL )&& (taosArrayGetSize(pScanInfo->pBlockList) > 0)) { numOfQTable += 1; } } @@ -1043,8 +1127,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { // no data exists, return directly. if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) { - tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader, pBlockInfo->uid, - pReader->idStr); + tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader, + pBlockInfo->uid, pReader->idStr); pResBlock->info.rows = 0; return 0; } @@ -1168,12 +1252,12 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI tBlockDataReset(pBlockData); STSchema* pSchema = getLatestTableSchema(pReader, uid); if (pSchema == NULL) { - tsdbDebug("%p table uid:%"PRIu64" has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); return code; } SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - TABLEID tid = {.suid = pReader->suid, .uid = uid}; + TABLEID tid = {.suid = pReader->suid, .uid = uid}; code = tBlockDataInit(pBlockData, &tid, pSchema, &pSup->colId[1], pSup->numOfCols - 1); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1598,7 +1682,8 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock // log the reason why load the datablock for profile if (loadDataBlock) { - tsdbDebug("%p uid:%" PRIu64 " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " + tsdbDebug("%p uid:%" PRIu64 + " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, @@ -1634,7 +1719,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64 - " - %" PRId64 ", uid:%"PRIu64", %s", + " - %" PRId64 ", uid:%" PRIu64 ", %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); @@ -1664,7 +1749,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, SVersionRange* pVerRange) { - int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1; while (1) { bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); @@ -1711,11 +1796,11 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ } pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); - if (pReader->pSchema == NULL) { + if (pReader->pSchema == NULL) { tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); } - return pReader->pSchema; + return pReader->pSchema; } static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { @@ -1756,7 +1841,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -1804,7 +1889,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1814,10 +1899,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1831,10 +1916,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } if (init) { - tRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1848,7 +1933,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1862,10 +1947,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1875,10 +1960,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } else { init = true; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1887,7 +1972,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1895,7 +1980,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1905,7 +1990,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -1916,16 +2001,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1933,10 +2018,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } } else { // not merge block data - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1948,7 +2033,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1956,7 +2041,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); } return TSDB_CODE_SUCCESS; @@ -1981,10 +2066,10 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1992,11 +2077,11 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2004,7 +2089,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } else { return TSDB_CODE_SUCCESS; @@ -2020,7 +2105,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SRowMerger merge = {0}; - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SArray* pDelList = pBlockScanInfo->delSkyline; @@ -2083,7 +2168,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2094,10 +2179,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2108,7 +2193,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); @@ -2116,7 +2201,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2135,10 +2220,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2153,7 +2238,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2167,11 +2252,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); } else { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2186,10 +2271,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tRowMerge(&merge, &fRow1); + tsdbRowMerge(&merge, &fRow1); } else { init = true; - code = tRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2200,7 +2285,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2208,7 +2293,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tRowMerge(&merge, &fRow); + tsdbRowMerge(&merge, &fRow); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2218,7 +2303,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2226,7 +2311,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return code; } @@ -2239,9 +2324,11 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea TSDBKEY startKey = {0}; if (ASCENDING_TRAVERSE(pReader->order)) { - startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + // startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->verRange.minVer}; } else { - startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + // startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->verRange.maxVer}; } int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); @@ -2348,7 +2435,8 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan w.ekey = pScanInfo->lastKey + step; } - tsdbDebug("init last block reader, window:%"PRId64"-%"PRId64", uid:%"PRIu64", %s", w.skey, w.ekey, pScanInfo->uid, pReader->idStr); + tsdbDebug("init last block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey, + pScanInfo->uid, pReader->idStr); int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, false, pReader->idStr); @@ -2382,16 +2470,16 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - STSRow* pTSRow = NULL; + SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(&merge, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2399,7 +2487,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tRowMergerClear(&merge); + tsdbRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } } @@ -2534,7 +2622,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { while (1) { bool hasBlockData = false; { - while (pBlockData->nRow > 0) { // find the first qualified row in data block + while (pBlockData->nRow > 0 && + pBlockData->uid == pBlockScanInfo->uid) { // find the first qualified row in data block if (isValidFileBlockRow(pBlockData, pDumpInfo, pBlockScanInfo, pReader)) { hasBlockData = true; break; @@ -2662,11 +2751,11 @@ _err: TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { bool asc = ASCENDING_TRAVERSE(pReader->order); -// TSKEY initialVal = asc? TSKEY_MIN:TSKEY_MAX; + // TSKEY initialVal = asc? TSKEY_MIN:TSKEY_MAX; - TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}, ikey = {.ts = TSKEY_INITIAL_VAL}; - bool hasKey = false, hasIKey = false; + bool hasKey = false, hasIKey = false; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { hasKey = true; @@ -2680,11 +2769,11 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } if (hasKey) { - if (hasIKey) { // has data in mem & imem + if (hasIKey) { // has data in mem & imem if (asc) { return key.ts <= ikey.ts ? key : ikey; - } else { - return key.ts <= ikey.ts ? ikey: key; + } else { + return key.ts <= ikey.ts ? ikey : key; } } else { // no data in imem return key; @@ -2705,13 +2794,19 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { - bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); + bool hasNext = false; + int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); + if (code) { + taosArrayDestroy(pIndexList); + return code; + } + if (!hasNext) { // no data files on disk break; } taosArrayClear(pIndexList); - int32_t code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList); + code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; @@ -2759,7 +2854,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -static void resetTableListIndex(SReaderStatus *pStatus) { +static void resetTableListIndex(SReaderStatus* pStatus) { STableUidList* pList = &pStatus->uidList; pList->currentIndex = 0; @@ -2919,7 +3014,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // update the last key for the corresponding table pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? pInfo->window.ekey : pInfo->window.skey; - tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " + tsdbDebug("%p uid:%" PRIu64 + " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->nRow, pBlock->minKey.ts, pBlock->maxKey.ts, pReader->idStr); @@ -2934,12 +3030,12 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { -// if (pStatus->pTableIter == NULL) { -// pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); -// if (pStatus->pTableIter == NULL) { -// return TSDB_CODE_SUCCESS; -// } -// } + // if (pStatus->pTableIter == NULL) { + // pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); + // if (pStatus->pTableIter == NULL) { + // return TSDB_CODE_SUCCESS; + // } + // } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; initMemDataIterator(*pBlockScanInfo, pReader); @@ -2963,9 +3059,16 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { } // set the correct start position in case of the first/last file block, according to the query time window -void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SDataBlk* pBlock = getCurrentBlock(pBlockIter); - +static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { + int64_t lastKey = ASCENDING_TRAVERSE(pReader->order) ? INT64_MIN : INT64_MAX; + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + if (pBlockInfo) { + STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pScanInfo) { + lastKey = pScanInfo->lastKey; + } + } SReaderStatus* pStatus = &pReader->status; SFileBlockDumpInfo* pDumpInfo = &pStatus->fBlockDumpInfo; @@ -2973,11 +3076,12 @@ void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { pDumpInfo->totalRows = pBlock->nRow; pDumpInfo->allDumped = false; pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->order) ? 0 : pBlock->nRow - 1; + pDumpInfo->lastKey = lastKey; } static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - int32_t code = moveToNextFile(pReader, &num); + int32_t code = moveToNextFile(pReader, &num); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3065,7 +3169,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } else { if (pReader->status.pCurrentFileset->nSttF > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } + + tBlockDataReset(pBlockData); resetDataBlockIterator(pBlockIter, pReader->order); resetTableListIndex(&pReader->status); goto _begin; @@ -3266,7 +3375,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key = {.ts = pRow->pTSRow->ts, .version = pRow->version}; + TSDBKEY key = TSDBROW_KEY(pRow); + if (outOfTimeWindow(key.ts, &pReader->window)) { pIter->hasVal = false; return NULL; @@ -3319,12 +3429,16 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe break; } - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + if (pRow->type == TSDBROW_ROW_FMT) { + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - tRowMergerAdd(pMerger, pRow, pTSchema); + tsdbRowMergerAdd(pMerger, pRow, pTSchema); + } else { // column format + tsdbRowMerge(pMerger, pRow); + } } return TSDB_CODE_SUCCESS; @@ -3339,7 +3453,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tRowMerge(pMerger, &fRow); + tsdbRowMerge(pMerger, &fRow); rowIndex += step; } @@ -3395,6 +3509,11 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + if (pFileBlockInfo == NULL) { + st = CHECK_FILEBLOCK_QUIT; + break; + } + checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3411,7 +3530,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tRowMerge(pMerger, &fRow1); + tsdbRowMerge(pMerger, &fRow1); } else { break; } @@ -3420,7 +3539,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc return TSDB_CODE_SUCCESS; } -int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; TSDBROW current = *pRow; @@ -3429,19 +3548,19 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { - *pTSRow = current.pTSRow; + *pResRow = *pRow; *freeTSRow = false; return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem pNextRow = getValidMemRow(pIter, pDelList, pReader); if (pNextRow == NULL) { - *pTSRow = current.pTSRow; + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } - if (current.pTSRow->ts != pNextRow->pTSRow->ts) { - *pTSRow = current.pTSRow; + if (TSDBROW_TS(¤t) != TSDBROW_TS(pNextRow)) { + *pResRow = current; *freeTSRow = false; return TSDB_CODE_SUCCESS; } @@ -3449,47 +3568,60 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } SRowMerger merge = {0}; - - // get the correct schema for data in memory terrno = 0; - STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); - if (pTSchema == NULL) { - return terrno; - } + int32_t code = 0; - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } + // start to merge duplicated rows + if (current.type == TSDBROW_ROW_FMT) { + // get the correct schema for data in memory + STSchema* pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(¤t), pReader, uid); + if (pTSchema == NULL) { + return terrno; + } - int32_t code = tRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); - if (code != TSDB_CODE_SUCCESS) { - return code; - } + if (pReader->pSchema == NULL) { + pReader->pSchema = pTSchema; + } - STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); - if (pTSchema1 == NULL) { - return terrno; - } + code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + STSchema* pTSchema1 = doGetSchemaForTSRow(TSDBROW_SVERSION(pNextRow), pReader, uid); + if (pTSchema1 == NULL) { + return terrno; + } - tRowMergerAdd(&merge, pNextRow, pTSchema1); + tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + } else { // let's merge rows in file block + code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + tsdbRowMerge(&merge, pNextRow); + } - code = doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tRowMergerGetRow(&merge, pTSRow); + code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } - tRowMergerClear(&merge); + pResRow->type = TSDBROW_ROW_FMT; + tsdbRowMergerClear(&merge); *freeTSRow = true; + return TSDB_CODE_SUCCESS; } int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { + SRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3498,7 +3630,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, piRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, piRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3509,7 +3641,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, pRow); + tsdbRowMerge(&merge, pRow); code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3519,7 +3651,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } else { STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3530,7 +3662,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tRowMerge(&merge, piRow); + tsdbRowMerge(&merge, piRow); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -3538,13 +3670,12 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } - int32_t code = tRowMergerGetRow(&merge, pTSRow); - tRowMergerClear(&merge); - + int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); + tsdbRowMergerClear(&merge); return code; } -int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, int64_t endKey, +int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); @@ -3574,13 +3705,14 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR int32_t code = TSDB_CODE_SUCCESS; if (ik.ts != k.ts) { if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts - code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { - code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } } else { // ik.ts == k.ts *freeTSRow = true; - code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + pResRow->type = TSDBROW_ROW_FMT; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3590,22 +3722,23 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pResRow, pReader, freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pResRow, pReader, freeTSRow); } return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, - STableBlockScanInfo* pScanInfo) { +int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) { int32_t outputRowIndex = pBlock->info.rows; int64_t uid = pScanInfo->uid; + int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); + SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); @@ -3624,7 +3757,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* if (colId == pSchema->columns[j].colId) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]); - tTSRowGetVal(pTSRow, pSchema, j, &colVal); + tRowGet(pTSRow, pSchema, j, &colVal); doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo); i += 1; j += 1; @@ -3703,17 +3836,22 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e SSDataBlock* pBlock = pReader->pResBlock; do { - STSRow* pTSRow = NULL; + // SRow* pTSRow = NULL; + TSDBROW row = {.type = -1}; bool freeTSRow = false; - tsdbGetNextRowInMem(pBlockScanInfo, pReader, &pTSRow, endKey, &freeTSRow); - if (pTSRow == NULL) { + tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow); + if (row.type == -1) { break; } - doAppendRowFromTSRow(pBlock, pReader, pTSRow, pBlockScanInfo); + if (row.type == TSDBROW_ROW_FMT) { + doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo); - if (freeTSRow) { - taosMemoryFree(pTSRow); + if (freeTSRow) { + taosMemoryFree(row.pTSRow); + } + } else { + doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow); } // no data in buffer, return immediately @@ -3878,49 +4016,15 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader; - pReader->status.pTableMap = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); + pReader->status.pTableMap = + createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables); if (pReader->status.pTableMap == NULL) { *ppReader = NULL; code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - if (numOfTables > 0) { - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - - if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { - code = doOpenReaderImpl(pReader); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - } else { - STsdbReader* pPrevReader = pReader->innerReader[0]; - STsdbReader* pNextReader = pReader->innerReader[1]; - - // we need only one row - pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->status.uidList = pReader->status.uidList; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pMemSchema = pReader->pMemSchema; - pPrevReader->pReadSnap = pReader->pReadSnap; - - pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->status.uidList = pReader->status.uidList; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pMemSchema = pReader->pMemSchema; - pNextReader->pReadSnap = pReader->pReadSnap; - - code = doOpenReaderImpl(pPrevReader); - if (code != TSDB_CODE_SUCCESS) { - goto _err; - } - } - } + pReader->suspended = true; tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -3928,7 +4032,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL _err: tsdbError("failed to create data reader, code:%s %s", tstrerror(code), idstr); tsdbReaderClose(pReader); - *ppReader = NULL; // reset the pointer value. + *ppReader = NULL; // reset the pointer value. return code; } @@ -3937,6 +4041,7 @@ void tsdbReaderClose(STsdbReader* pReader) { return; } + tsdbAcquireReader(pReader); { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { STsdbReader* p = pReader->innerReader[0]; @@ -3974,7 +4079,7 @@ void tsdbReaderClose(STsdbReader* pReader) { } taosMemoryFree(pSupInfo->colId); - tBlockDataDestroy(&pReader->status.fileBlockData, true); + tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); @@ -3996,7 +4101,13 @@ void tsdbReaderClose(STsdbReader* pReader) { pReader->pDelIdx = NULL; } - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); + qTrace("tsdb/reader-close: %p, untake snapshot", pReader); + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, true); + pReader->pReadSnap = NULL; + + tsdbReleaseReader(pReader); + + tsdbUninitReaderLock(pReader); taosMemoryFree(pReader->status.uidList.tableUidList); SIOCostSummary* pCost = &pReader->cost; @@ -4017,7 +4128,8 @@ void tsdbReaderClose(STsdbReader* pReader) { " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 - ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,initDelSkylineIterTime:%.2f ms, %s", + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,initDelSkylineIterTime:%.2f " + "ms, %s", pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, @@ -4033,6 +4145,194 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pReader); } +int32_t tsdbReaderSuspend(STsdbReader* pReader) { + int32_t code = 0; + + // save reader's base state & reset top state to be reconstructed from base state + SReaderStatus* pStatus = &pReader->status; + STableBlockScanInfo* pBlockScanInfo = NULL; + + if (pStatus->loadFromFile) { + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + if (pBlockInfo != NULL) { + pBlockScanInfo = + *(STableBlockScanInfo**)taosHashGet(pStatus->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + if (pBlockScanInfo == NULL) { + code = TSDB_CODE_INVALID_PARA; + tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", pBlockInfo->uid, + taosHashGetSize(pReader->status.pTableMap), pReader->idStr); + goto _err; + } + } else { + pBlockScanInfo = *pStatus->pTableIter; + } + + tsdbDataFReaderClose(&pReader->pFileReader); + + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + // pInfo->lastKey = ts; + } + } else { + // resetDataBlockScanInfo excluding lastKey + STableBlockScanInfo** p = NULL; + + while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; + + pInfo->iterInit = false; + pInfo->iter.hasVal = false; + pInfo->iiter.hasVal = false; + + if (pInfo->iter.iter != NULL) { + pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter); + } + + if (pInfo->iiter.iter != NULL) { + pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter); + } + + pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); + // pInfo->lastKey = ts; + } + + pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; + if (pBlockScanInfo) { + // save lastKey to restore memory iterator + STimeWindow w = pReader->pResBlock->info.window; + pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? w.ekey : w.skey; + + // reset current current table's data block scan info, + pBlockScanInfo->iterInit = false; + pBlockScanInfo->iter.hasVal = false; + pBlockScanInfo->iiter.hasVal = false; + if (pBlockScanInfo->iter.iter != NULL) { + pBlockScanInfo->iter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iter.iter); + } + + if (pBlockScanInfo->iiter.iter != NULL) { + pBlockScanInfo->iiter.iter = tsdbTbDataIterDestroy(pBlockScanInfo->iiter.iter); + } + + pBlockScanInfo->pBlockList = taosArrayDestroy(pBlockScanInfo->pBlockList); + tMapDataClear(&pBlockScanInfo->mapData); + // TODO: keep skyline for reuse + pBlockScanInfo->delSkyline = taosArrayDestroy(pBlockScanInfo->delSkyline); + } + } + + tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false); + pReader->pReadSnap = NULL; + + pReader->suspended = true; + + tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, + pReader->idStr); + return code; + +_err: + tsdbError("failed to suspend data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + +static int32_t tsdbSetQueryReseek(void* pQHandle) { + int32_t code = 0; + STsdbReader* pReader = pQHandle; + + code = tsdbTryAcquireReader(pReader); + if (code == 0) { + if (pReader->suspended) { + tsdbReleaseReader(pReader); + return code; + } + + tsdbReaderSuspend(pReader); + + tsdbReleaseReader(pReader); + + return code; + } else if (code == EBUSY) { + return TSDB_CODE_VND_QUERY_BUSY; + } else { + terrno = TAOS_SYSTEM_ERROR(code); + return TSDB_CODE_FAILED; + } +} + +int32_t tsdbReaderResume(STsdbReader* pReader) { + int32_t code = 0; + + STableBlockScanInfo** pBlockScanInfo = pReader->status.pTableIter; + + // restore reader's state + // task snapshot + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + if (numOfTables > 0) { + qTrace("tsdb/reader: %p, take snapshot", pReader); + code = tsdbTakeReadSnap(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); + if (code != TSDB_CODE_SUCCESS) { + goto _err; + } + + if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) { + code = doOpenReaderImpl(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + STsdbReader* pPrevReader = pReader->innerReader[0]; + STsdbReader* pNextReader = pReader->innerReader[1]; + + // we need only one row + pPrevReader->capacity = 1; + pPrevReader->status.pTableMap = pReader->status.pTableMap; + pPrevReader->status.uidList = pReader->status.uidList; + pPrevReader->pSchema = pReader->pSchema; + pPrevReader->pMemSchema = pReader->pMemSchema; + pPrevReader->pReadSnap = pReader->pReadSnap; + + pNextReader->capacity = 1; + pNextReader->status.pTableMap = pReader->status.pTableMap; + pNextReader->status.uidList = pReader->status.uidList; + pNextReader->pSchema = pReader->pSchema; + pNextReader->pMemSchema = pReader->pMemSchema; + pNextReader->pReadSnap = pReader->pReadSnap; + + code = doOpenReaderImpl(pPrevReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + pReader->suspended = false; + + tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, + pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); + return code; + +_err: + tsdbError("failed to resume data reader, code:%s %s", tstrerror(code), pReader->idStr); + return code; +} + static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; @@ -4067,10 +4367,25 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { return false; } - if (pReader->step == 0 && pReader->innerReader[0] != NULL) { + SReaderStatus* pStatus = &pReader->status; + + int32_t code = tsdbAcquireReader(pReader); + qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); + + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + + if (pReader->innerReader[0] != NULL && pReader->step == 0) { bool ret = doTsdbNextDataBlock(pReader->innerReader[0]); pReader->step = EXTERNAL_ROWS_PREV; if (ret) { + pStatus = &pReader->innerReader[0]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + return ret; } } @@ -4089,6 +4404,11 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { bool ret = doTsdbNextDataBlock(pReader); if (ret) { + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + return ret; } @@ -4103,10 +4423,19 @@ bool tsdbNextDataBlock(STsdbReader* pReader) { ret = doTsdbNextDataBlock(pReader->innerReader[1]); pReader->step = EXTERNAL_ROWS_NEXT; if (ret) { + pStatus = &pReader->innerReader[1]->status; + if (pStatus->composedDataBlock) { + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + } + return ret; } } + qTrace("tsdb/read: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + return false; } @@ -4234,12 +4563,7 @@ STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, co } static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; - - if (pStatus->composedDataBlock) { - return pReader->pResBlock; - } - + SReaderStatus* pStatus = &pReader->status; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { @@ -4248,7 +4572,7 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { - tBlockDataDestroy(&pStatus->fileBlockData, 1); + tBlockDataDestroy(&pStatus->fileBlockData); terrno = code; return NULL; } @@ -4258,19 +4582,41 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { } SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { + STsdbReader* pTReader = pReader; if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_PREV) { - return doRetrieveDataBlock(pReader->innerReader[0]); + pTReader = pReader->innerReader[0]; } else if (pReader->step == EXTERNAL_ROWS_NEXT) { - return doRetrieveDataBlock(pReader->innerReader[1]); + pTReader = pReader->innerReader[1]; } } - return doRetrieveDataBlock(pReader); + SReaderStatus* pStatus = &pTReader->status; + if (pStatus->composedDataBlock) { + return pTReader->pResBlock; + } + + SSDataBlock* ret = doRetrieveDataBlock(pTReader); + + qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); + tsdbReleaseReader(pReader); + + return ret; } int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { + qTrace("tsdb/reader-reset: %p, take read mutex", pReader); + tsdbAcquireReader(pReader); + + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->pReadSnap == NULL) { + tsdbDebug("tsdb reader reset return %p", pReader->pReadSnap); + + tsdbReleaseReader(pReader); + return TSDB_CODE_SUCCESS; } @@ -4310,6 +4656,9 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { if (code != TSDB_CODE_SUCCESS) { tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); + + tsdbReleaseReader(pReader); + return code; } } @@ -4319,6 +4668,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr); + tsdbReleaseReader(pReader); + return code; } @@ -4330,8 +4681,14 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa int32_t code = TSDB_CODE_SUCCESS; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; + pTableBlockInfo->numOfVgroups = 1; // find the start data block in file + + tsdbAcquireReader(pReader); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } SReaderStatus* pStatus = &pReader->status; STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg; @@ -4393,7 +4750,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, // pReader->pFileGroup->fid, pReader->idStr); } - + tsdbReleaseReader(pReader); return code; } @@ -4401,6 +4758,11 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { int64_t rows = 0; SReaderStatus* pStatus = &pReader->status; + tsdbAcquireReader(pReader); + if (pReader->suspended) { + tsdbReaderResume(pReader); + } + pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); while (pStatus->pTableIter != NULL) { @@ -4426,6 +4788,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); } + tsdbReleaseReader(pReader); + return rows; } @@ -4467,66 +4831,92 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } -int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr) { - int32_t code = 0; +int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + SVersionRange* pRange = &pReader->verRange; // alloc - *ppSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap)); - if (*ppSnap == NULL) { + STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(*pSnap)); + if (pSnap == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } // lock - code = taosThreadRwlockRdlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockRdlock(&pTsdb->rwLock); // take snapshot - (*ppSnap)->pMem = pTsdb->mem; - (*ppSnap)->pIMem = pTsdb->imem; + if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) { + pSnap->pMem = pTsdb->mem; + pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode)); + if (pSnap->pNode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pNode->pQHandle = pReader; + pSnap->pNode->reseek = reseek; - if ((*ppSnap)->pMem) { - tsdbRefMemTable((*ppSnap)->pMem); + tsdbRefMemTable(pTsdb->mem, pSnap->pNode); } - if ((*ppSnap)->pIMem) { - tsdbRefMemTable((*ppSnap)->pIMem); + if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) { + pSnap->pIMem = pTsdb->imem; + pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode)); + if (pSnap->pINode == NULL) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + pSnap->pINode->pQHandle = pReader; + pSnap->pINode->reseek = reseek; + + tsdbRefMemTable(pTsdb->imem, pSnap->pINode); } // fs - code = tsdbFSRef(pTsdb, &(*ppSnap)->fs); + code = tsdbFSRef(pTsdb, &pSnap->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); goto _exit; } // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { - code = TAOS_SYSTEM_ERROR(code); - goto _exit; - } + taosThreadRwlockUnlock(&pTsdb->rwLock); + + tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); - tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); _exit: + if (code) { + *ppSnap = NULL; + if (pSnap) { + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); + taosMemoryFree(pSnap); + } + } else { + *ppSnap = pSnap; + } return code; } -void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) { +void tsdbUntakeReadSnap(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) { + STsdb* pTsdb = pReader->pTsdb; + if (pSnap) { if (pSnap->pMem) { - tsdbUnrefMemTable(pSnap->pMem); + tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive); } if (pSnap->pIMem) { - tsdbUnrefMemTable(pSnap->pIMem); + tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } tsdbFSUnref(pTsdb, &pSnap->fs); + if (pSnap->pNode) taosMemoryFree(pSnap->pNode); + if (pSnap->pINode) taosMemoryFree(pSnap->pINode); taosMemoryFree(pSnap); } - tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); + tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); } diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index e8181f922f0262a72f2f40bed319a17008738cbb..1a98134d70c486259a659cb9771152333835b8c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -15,457 +15,10 @@ #include "tsdb.h" -extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg); extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg); -// STsdbDataIter2 ======================================== -#define TSDB_MEM_TABLE_DATA_ITER 0 -#define TSDB_DATA_FILE_DATA_ITER 1 -#define TSDB_STT_FILE_DATA_ITER 2 -#define TSDB_TOMB_FILE_DATA_ITER 3 - -typedef struct STsdbDataIter2 STsdbDataIter2; -typedef struct STsdbFilterInfo STsdbFilterInfo; - -typedef struct { - int64_t suid; - int64_t uid; - SDelData delData; -} SDelInfo; - -struct STsdbDataIter2 { - STsdbDataIter2* next; - SRBTreeNode rbtn; - - int32_t type; - SRowInfo rowInfo; - SDelInfo delInfo; - union { - // TSDB_MEM_TABLE_DATA_ITER - struct { - SMemTable* pMemTable; - } mIter; - - // TSDB_DATA_FILE_DATA_ITER - struct { - SDataFReader* pReader; - SArray* aBlockIdx; // SArray - SMapData mDataBlk; - SBlockData bData; - int32_t iBlockIdx; - int32_t iDataBlk; - int32_t iRow; - } dIter; - - // TSDB_STT_FILE_DATA_ITER - struct { - SDataFReader* pReader; - int32_t iStt; - SArray* aSttBlk; - SBlockData bData; - int32_t iSttBlk; - int32_t iRow; - } sIter; - // TSDB_TOMB_FILE_DATA_ITER - struct { - SDelFReader* pReader; - SArray* aDelIdx; - SArray* aDelData; - int32_t iDelIdx; - int32_t iDelData; - } tIter; - }; -}; - -#define TSDB_FILTER_FLAG_BY_VERSION 0x1 -struct STsdbFilterInfo { - int32_t flag; - int64_t sver; - int64_t ever; -}; - -#define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2*)(((char*)pNode) - offsetof(STsdbDataIter2, rbtn))) - -/* open */ -static int32_t tsdbOpenDataFileDataIter(SDataFReader* pReader, STsdbDataIter2** ppIter) { - int32_t code = 0; - int32_t lino = 0; - - // create handle - STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); - if (pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pIter->type = TSDB_DATA_FILE_DATA_ITER; - pIter->dIter.pReader = pReader; - if ((pIter->dIter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pIter->dIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iBlockIdx = 0; - pIter->dIter.iDataBlk = 0; - pIter->dIter.iRow = 0; - - // read data - code = tsdbReadBlockIdx(pReader, pIter->dIter.aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pIter->dIter.aBlockIdx) == 0) goto _clear; - -_exit: - if (code) { - if (pIter) { - _clear: - tBlockDataDestroy(&pIter->dIter.bData, 1); - taosArrayDestroy(pIter->dIter.aBlockIdx); - taosMemoryFree(pIter); - pIter = NULL; - } - } - *ppIter = pIter; - return code; -} - -static int32_t tsdbOpenSttFileDataIter(SDataFReader* pReader, int32_t iStt, STsdbDataIter2** ppIter) { - int32_t code = 0; - int32_t lino = 0; - - // create handle - STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); - if (pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pIter->type = TSDB_STT_FILE_DATA_ITER; - pIter->sIter.pReader = pReader; - pIter->sIter.iStt = iStt; - pIter->sIter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pIter->sIter.aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pIter->sIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->sIter.iSttBlk = 0; - pIter->sIter.iRow = 0; - - // read data - code = tsdbReadSttBlk(pReader, iStt, pIter->sIter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pIter->sIter.aSttBlk) == 0) goto _clear; - -_exit: - if (code) { - if (pIter) { - _clear: - taosArrayDestroy(pIter->sIter.aSttBlk); - tBlockDataDestroy(&pIter->sIter.bData, 1); - taosMemoryFree(pIter); - pIter = NULL; - } - } - *ppIter = pIter; - return code; -} - -static int32_t tsdbOpenTombFileDataIter(SDelFReader* pReader, STsdbDataIter2** ppIter) { - int32_t code = 0; - int32_t lino = 0; - - STsdbDataIter2* pIter = (STsdbDataIter2*)taosMemoryCalloc(1, sizeof(*pIter)); - if (pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - pIter->type = TSDB_TOMB_FILE_DATA_ITER; - - pIter->tIter.pReader = pReader; - if ((pIter->tIter.aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - if ((pIter->tIter.aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbReadDelIdx(pReader, pIter->tIter.aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pIter->tIter.aDelIdx) == 0) goto _clear; - - pIter->tIter.iDelIdx = 0; - pIter->tIter.iDelData = 0; - -_exit: - if (code) { - if (pIter) { - _clear: - taosArrayDestroy(pIter->tIter.aDelIdx); - taosArrayDestroy(pIter->tIter.aDelData); - taosMemoryFree(pIter); - pIter = NULL; - } - } - *ppIter = pIter; - return code; -} - -/* close */ -static void tsdbCloseDataFileDataIter(STsdbDataIter2* pIter) { - tBlockDataDestroy(&pIter->dIter.bData, 1); - tMapDataClear(&pIter->dIter.mDataBlk); - taosArrayDestroy(pIter->dIter.aBlockIdx); - taosMemoryFree(pIter); -} - -static void tsdbCloseSttFileDataIter(STsdbDataIter2* pIter) { - tBlockDataDestroy(&pIter->sIter.bData, 1); - taosArrayDestroy(pIter->sIter.aSttBlk); - taosMemoryFree(pIter); -} - -static void tsdbCloseTombFileDataIter(STsdbDataIter2* pIter) { - taosArrayDestroy(pIter->tIter.aDelData); - taosArrayDestroy(pIter->tIter.aDelIdx); - taosMemoryFree(pIter); -} - -static void tsdbCloseDataIter2(STsdbDataIter2* pIter) { - if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) { - ASSERT(0); - } else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) { - tsdbCloseDataFileDataIter(pIter); - } else if (pIter->type == TSDB_STT_FILE_DATA_ITER) { - tsdbCloseSttFileDataIter(pIter); - } else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) { - tsdbCloseTombFileDataIter(pIter); - } else { - ASSERT(0); - } -} - -/* cmpr */ -static int32_t tsdbDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pNode2) { - STsdbDataIter2* pIter1 = TSDB_RBTN_TO_DATA_ITER(pNode1); - STsdbDataIter2* pIter2 = TSDB_RBTN_TO_DATA_ITER(pNode2); - return tRowInfoCmprFn(&pIter1->rowInfo, &pIter2->rowInfo); -} - -/* seek */ - -/* iter next */ -static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { - int32_t code = 0; - int32_t lino = 0; - - for (;;) { - while (pIter->dIter.iRow < pIter->dIter.bData.nRow) { - if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { - if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver || - pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) { - pIter->dIter.iRow++; - continue; - } - } - } - - pIter->rowInfo.suid = pIter->dIter.bData.suid; - pIter->rowInfo.uid = pIter->dIter.bData.uid; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->dIter.bData, pIter->dIter.iRow); - pIter->dIter.iRow++; - goto _exit; - } - - for (;;) { - while (pIter->dIter.iDataBlk < pIter->dIter.mDataBlk.nItem) { - SDataBlk dataBlk; - tMapDataGetItemByIdx(&pIter->dIter.mDataBlk, pIter->dIter.iDataBlk, &dataBlk, tGetDataBlk); - - // filter - if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { - if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) { - pIter->dIter.iDataBlk++; - continue; - } - } - } - - code = tsdbReadDataBlockEx(pIter->dIter.pReader, &dataBlk, &pIter->dIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iDataBlk++; - pIter->dIter.iRow = 0; - - break; - } - - if (pIter->dIter.iRow < pIter->dIter.bData.nRow) break; - - for (;;) { - if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) { - SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); - - code = tsdbReadDataBlk(pIter->dIter.pReader, pBlockIdx, &pIter->dIter.mDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->dIter.iBlockIdx++; - pIter->dIter.iDataBlk = 0; - - break; - } else { - pIter->rowInfo = (SRowInfo){0}; - goto _exit; - } - } - } - } - -_exit: - if (code) { - tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { - int32_t code = 0; - int32_t lino = 0; - - for (;;) { - while (pIter->sIter.iRow < pIter->sIter.bData.nRow) { - if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { - if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] || - pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) { - pIter->sIter.iRow++; - continue; - } - } - } - - pIter->rowInfo.suid = pIter->sIter.bData.suid; - pIter->rowInfo.uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; - pIter->rowInfo.row = tsdbRowFromBlockData(&pIter->sIter.bData, pIter->sIter.iRow); - pIter->sIter.iRow++; - goto _exit; - } - - for (;;) { - if (pIter->sIter.iSttBlk < taosArrayGetSize(pIter->sIter.aSttBlk)) { - SSttBlk* pSttBlk = taosArrayGet(pIter->sIter.aSttBlk, pIter->sIter.iSttBlk); - - if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { - if (pFilterInfo->sver > pSttBlk->maxVer || pFilterInfo->ever < pSttBlk->minVer) { - pIter->sIter.iSttBlk++; - continue; - } - } - } - - code = tsdbReadSttBlockEx(pIter->sIter.pReader, pIter->sIter.iStt, pSttBlk, &pIter->sIter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->sIter.iRow = 0; - pIter->sIter.iSttBlk++; - break; - } else { - pIter->rowInfo = (SRowInfo){0}; - goto _exit; - } - } - } - -_exit: - if (code) { - tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { - int32_t code = 0; - int32_t lino = 0; - - for (;;) { - while (pIter->tIter.iDelData < taosArrayGetSize(pIter->tIter.aDelData)) { - SDelData* pDelData = taosArrayGet(pIter->tIter.aDelData, pIter->tIter.iDelData); - - if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { - if (pFilterInfo->sver > pDelData->version || pFilterInfo->ever < pDelData->version) { - pIter->tIter.iDelData++; - continue; - } - } - } - - pIter->delInfo.delData = *pDelData; - pIter->tIter.iDelData++; - goto _exit; - } - - for (;;) { - if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) { - SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx); - - code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->delInfo.suid = pDelIdx->suid; - pIter->delInfo.uid = pDelIdx->uid; - pIter->tIter.iDelData = 0; - pIter->tIter.iDelIdx++; - break; - } else { - pIter->delInfo = (SDelInfo){0}; - goto _exit; - } - } - } - -_exit: - if (code) { - tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbDataIterNext2(STsdbDataIter2* pIter, STsdbFilterInfo* pFilterInfo) { - int32_t code = 0; - - if (pIter->type == TSDB_MEM_TABLE_DATA_ITER) { - ASSERT(0); - return code; - } else if (pIter->type == TSDB_DATA_FILE_DATA_ITER) { - return tsdbDataFileDataIterNext(pIter, pFilterInfo); - } else if (pIter->type == TSDB_STT_FILE_DATA_ITER) { - return tsdbSttFileDataIterNext(pIter, pFilterInfo); - } else if (pIter->type == TSDB_TOMB_FILE_DATA_ITER) { - return tsdbTombFileDataIterNext(pIter, pFilterInfo); - } else { - ASSERT(0); - return code; - } -} - -/* get */ - // STsdbSnapReader ======================================== struct STsdbSnapReader { STsdb* pTsdb; @@ -656,7 +209,7 @@ static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { } SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData; - pHdr->type = SNAP_DATA_TSDB; + pHdr->type = pReader->type; pHdr->size = size; memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]); @@ -732,6 +285,8 @@ static int32_t tsdbSnapReadTimeSeriesData(STsdbSnapReader* pReader, uint8_t** pp } if (pReader->bData.nRow > 0) { + ASSERT(pReader->bData.suid || pReader->bData.uid); + code = tsdbSnapCmprData(pReader, ppData); TSDB_CHECK_CODE(code, lino, _exit); } @@ -773,10 +328,6 @@ static int32_t tsdbSnapCmprTombData(STsdbSnapReader* pReader, uint8_t** ppData) _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pReader->pTsdb->pVnode), __func__, lino, tstrerror(code)); - if (pData) { - taosMemoryFree(pData); - pData = NULL; - } } *ppData = pData; return code; @@ -849,7 +400,7 @@ static int32_t tsdbSnapReadTombData(STsdbSnapReader* pReader, uint8_t** ppData) } while (pDelInfo && pDelInfo->suid == pReader->tbid.suid && pDelInfo->uid == pReader->tbid.uid) { - if (taosArrayPush(pReader->aDelData, &pDelInfo->delData) < 0) { + if (taosArrayPush(pReader->aDelData, &pDelInfo->delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } @@ -907,7 +458,7 @@ _exit: tsdbError("vgId:%d %s failed at line %d since %s, sver:%" PRId64 " ever:%" PRId64 " type:%d", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code), sver, ever, type); if (pReader) { - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); tsdbFSUnref(pTsdb, &pReader->fs); taosMemoryFree(pReader); pReader = NULL; @@ -946,7 +497,7 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } - tBlockDataDestroy(&pReader->bData, 1); + tBlockDataDestroy(&pReader->bData); // other tDestroyTSchema(pReader->skmTable.pTSchema); @@ -1316,7 +867,7 @@ static int32_t tsdbSnapWriteFileDataStart(STsdbSnapWriter* pWriter, int32_t fid) TSDB_CHECK_CODE(code, lino, _exit); if (pWriter->pSIter) { - code = tsdbSttFileDataIterNext(pWriter->pSIter, NULL); + code = tsdbDataIterNext2(pWriter->pSIter, NULL); TSDB_CHECK_CODE(code, lino, _exit); // add to tree @@ -1697,7 +1248,7 @@ static int32_t tsdbSnapWriteDelTableData(STsdbSnapWriter* pWriter, TABLEID* pId, SDelData delData; n += tGetDelData(pData + n, &delData); - if (taosArrayPush(pWriter->aDelData, &delData) < 0) { + if (taosArrayPush(pWriter->aDelData, &delData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } @@ -1861,10 +1412,11 @@ _exit: if (code) { tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); if (pWriter) { - tBlockDataDestroy(&pWriter->sData, 1); - tBlockDataDestroy(&pWriter->bData, 1); - tBlockDataDestroy(&pWriter->inData, 1); + tBlockDataDestroy(&pWriter->sData); + tBlockDataDestroy(&pWriter->bData); + tBlockDataDestroy(&pWriter->inData); tsdbFSDestroy(&pWriter->fs); + taosMemoryFree(pWriter); pWriter = NULL; } } else { @@ -1928,13 +1480,13 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { taosArrayDestroy(pWriter->aDelIdx); // SNAP_DATA_TSDB - tBlockDataDestroy(&pWriter->sData, 1); - tBlockDataDestroy(&pWriter->bData, 1); + tBlockDataDestroy(&pWriter->sData); + tBlockDataDestroy(&pWriter->bData); taosArrayDestroy(pWriter->aSttBlk); tMapDataClear(&pWriter->mDataBlk); taosArrayDestroy(pWriter->aBlockIdx); tDestroyTSchema(pWriter->skmTable.pTSchema); - tBlockDataDestroy(&pWriter->inData, 1); + tBlockDataDestroy(&pWriter->inData); for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 853f0bcc2129e1a3aad964f8436d8b46a8f7a9c7..c323ae1532a3f8fe97fdcddaae020723bda88c94 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -23,7 +23,7 @@ void tMapDataReset(SMapData *pMapData) { } void tMapDataClear(SMapData *pMapData) { - tFree((uint8_t *)pMapData->aOffset); + tFree(pMapData->aOffset); tFree(pMapData->pData); pMapData->pData = NULL; pMapData->aOffset = NULL; @@ -116,12 +116,7 @@ int32_t tMapDataToArray(SMapData *pMapData, int32_t itemSize, int32_t (*tGetItem } _exit: - if (code) { - *ppArray = NULL; - if (pArray) taosArrayDestroy(pArray); - } else { - *ppArray = pArray; - } + *ppArray = pArray; return code; } @@ -572,9 +567,9 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * ASSERT(iCol > 0); - if (pRow->type == 0) { - tTSRowGetVal(pRow->pTSRow, pTSchema, iCol, pColVal); - } else if (pRow->type == 1) { + if (pRow->type == TSDBROW_ROW_FMT) { + tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); + } else if (pRow->type == TSDBROW_COL_FMT) { SColData *pColData; tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData); @@ -589,60 +584,61 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } -// int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) { -// int32_t n = 0; - -// n += tPutI64(p, pRow->version); -// if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len); -// n += pRow->pTSRow->len; - -// return n; -// } - int32_t tsdbRowCmprFn(const void *p1, const void *p2) { return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); } // STSDBRowIter ====================================================== -void tsdbRowIterInit(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) { + int32_t code = 0; + pIter->pRow = pRow; - if (pRow->type == 0) { - ASSERT(pTSchema); - pIter->pTSchema = pTSchema; - pIter->i = 1; - } else if (pRow->type == 1) { - pIter->pTSchema = NULL; - pIter->i = 0; + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowIterOpen(pRow->pTSRow, pTSchema, &pIter->pIter); + if (code) goto _exit; + } else if (pRow->type == TSDBROW_COL_FMT) { + pIter->iColData = 0; } else { ASSERT(0); } + +_exit: + return code; } -SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { - if (pIter->pRow->type == 0) { - if (pIter->i < pIter->pTSchema->numOfCols) { - tTSRowGetVal(pIter->pRow->pTSRow, pIter->pTSchema, pIter->i, &pIter->colVal); - pIter->i++; +void tsdbRowClose(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + tRowIterClose(&pIter->pIter); + } +} - return &pIter->colVal; +SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { + if (pIter->pRow->type == TSDBROW_ROW_FMT) { + return tRowIterNext(pIter->pIter); + } else if (pIter->pRow->type == TSDBROW_COL_FMT) { + if (pIter->iColData == 0) { + pIter->cv = COL_VAL_VALUE(PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, + (SValue){.val = pIter->pRow->pBlockData->aTSKEY[pIter->pRow->iRow]}); + ++pIter->iColData; + return &pIter->cv; } - } else { - if (pIter->i < pIter->pRow->pBlockData->nColData) { - SColData *pColData = tBlockDataGetColDataByIdx(pIter->pRow->pBlockData, pIter->i); - tColDataGetValue(pColData, pIter->pRow->iRow, &pIter->colVal); - pIter->i++; - - return &pIter->colVal; + if (pIter->iColData < pIter->pRow->pBlockData->nColData) { + tColDataGetValue(&pIter->pRow->pBlockData->aColData[pIter->iColData], pIter->pRow->iRow, &pIter->cv); + ++pIter->iColData; + return &pIter->cv; + } else { + return NULL; } + } else { + ASSERT(0); + return NULL; // suppress error report by compiler } - - return NULL; } // SRowMerger ====================================================== -int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -709,7 +705,7 @@ _exit: return code; } -int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -794,7 +790,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { return code; } -int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -845,7 +841,7 @@ _exit: return code; } -void tRowMergerClear(SRowMerger *pMerger) { +void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (IS_VAR_DATA_TYPE(pTColVal->type)) { @@ -856,7 +852,7 @@ void tRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } -int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { +int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; @@ -921,12 +917,8 @@ _exit: return code; } -int32_t tRowMergerGetRow(SRowMerger *pMerger, STSRow **ppRow) { - int32_t code = 0; - - code = tdSTSRowNew(pMerger->pArray, pMerger->pTSchema, ppRow); - - return code; +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { + return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } /* @@ -1058,14 +1050,13 @@ static int32_t tsdbMergeSkyline(SArray *pSkyline1, SArray *pSkyline2, SArray *pS return code; } - int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, SArray *pSkyline) { int32_t code = 0; SDelData *pDelData; int32_t midx; taosArrayClear(pSkyline); - if (sidx == eidx) { + if (sidx == eidx) { TSDBKEY *pItem1 = taosArrayGet(aSkyline, sidx * 2); TSDBKEY *pItem2 = taosArrayGet(aSkyline, sidx * 2 + 1); taosArrayPush(pSkyline, &pItem1); @@ -1098,14 +1089,14 @@ int32_t tsdbBuildDeleteSkylineImpl(SArray *aSkyline, int32_t sidx, int32_t eidx, return code; } - int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SArray *aSkyline) { SDelData *pDelData; - int32_t code = 0; - int32_t dataNum = eidx - sidx + 1; - SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY)); - SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); - + int32_t code = 0; + int32_t dataNum = eidx - sidx + 1; + SArray *aTmpSkyline = taosArrayInit(dataNum * 2, sizeof(TSDBKEY)); + SArray *pSkyline = taosArrayInit(dataNum * 2, POINTER_BYTES); + + taosArrayClear(aSkyline); for (int32_t i = sidx; i <= eidx; ++i) { pDelData = (SDelData *)taosArrayGet(aDelData, i); taosArrayPush(aTmpSkyline, &(TSDBKEY){.ts = pDelData->sKey, .version = pDelData->version}); @@ -1117,8 +1108,8 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr int32_t skylineNum = taosArrayGetSize(pSkyline); for (int32_t i = 0; i < skylineNum; ++i) { - TSDBKEY *p = taosArrayGetP(pSkyline, i); - taosArrayPush(aSkyline, p); + TSDBKEY *p = taosArrayGetP(pSkyline, i); + taosArrayPush(aSkyline, p); } _clear: @@ -1179,27 +1170,49 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; - pBlockData->aColData = taosArrayInit(0, sizeof(SColData)); - if (pBlockData->aColData == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pBlockData->aColData = NULL; _exit: return code; } -void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { - tFree((uint8_t *)pBlockData->aUid); - tFree((uint8_t *)pBlockData->aVersion); - tFree((uint8_t *)pBlockData->aTSKEY); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); - pBlockData->aUid = NULL; - pBlockData->aVersion = NULL; - pBlockData->aTSKEY = NULL; - pBlockData->aColData = NULL; +void tBlockDataDestroy(SBlockData *pBlockData) { + tFree(pBlockData->aUid); + tFree(pBlockData->aVersion); + tFree(pBlockData->aTSKEY); + + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + + if (pBlockData->aColData) { + taosMemoryFree(pBlockData->aColData); + pBlockData->aColData = NULL; + } } +static int32_t tBlockDataAdjustColData(SBlockData *pBlockData, int32_t nColData) { + int32_t code = 0; + + if (pBlockData->nColData > nColData) { + for (int32_t i = nColData; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + } else if (pBlockData->nColData < nColData) { + SColData *aColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SBlockData) * nColData); + if (aColData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pBlockData->aColData = aColData; + memset(&pBlockData->aColData[pBlockData->nColData], 0, sizeof(SBlockData) * (nColData - pBlockData->nColData)); + } + pBlockData->nColData = nColData; + +_exit: + return code; +} int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid) { int32_t code = 0; @@ -1209,37 +1222,43 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pBlockData->uid = pId->uid; pBlockData->nRow = 0; - pBlockData->nColData = 0; if (aCid) { + code = tBlockDataAdjustColData(pBlockData, nCid); + if (code) goto _exit; + int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - while (pTColumn && pTColumn->colId < aCid[iCid]) { - iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; + if (ASSERTS(pTColumn != NULL, "invalid input param")) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; } - if (pTColumn == NULL) { - break; - } else if (pTColumn->colId == aCid[iCid]) { - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); - + while (pTColumn->colId < aCid[iCid]) { iColumn++; - pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; + ASSERT(iColumn < pTSchema->numOfCols); + pTColumn = &pTSchema->columns[iColumn]; + } + + if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) { + code = TSDB_CODE_INVALID_PARA; + goto _exit; } + + tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + + iColumn++; + pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; } } else { - for (int32_t iColumn = 1; iColumn < pTSchema->numOfCols; iColumn++) { - STColumn *pTColumn = &pTSchema->columns[iColumn]; - - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, pTSchema->numOfCols - 1); + if (code) goto _exit; - tColDataInit(pColData, pTColumn->colId, pTColumn->type, (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + STColumn *pTColumn = &pTSchema->columns[iColData + 1]; + tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, + (pTColumn->flags & COL_SMA_ON) ? 1 : 0); } } @@ -1251,7 +1270,6 @@ void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; - pBlockData->nColData = 0; } void tBlockDataClear(SBlockData *pBlockData) { @@ -1259,211 +1277,43 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataClear(pColData); - } -} - -int32_t tBlockDataAddColData(SBlockData *pBlockData, SColData **ppColData) { - int32_t code = 0; - SColData *pColData = NULL; - - if (pBlockData->nColData >= taosArrayGetSize(pBlockData->aColData)) { - if (taosArrayPush(pBlockData->aColData, &((SColData){0})) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + tColDataClear(tBlockDataGetColDataByIdx(pBlockData, iColData)); } - pColData = (SColData *)taosArrayGet(pBlockData->aColData, pBlockData->nColData); - - pBlockData->nColData++; - - *ppColData = pColData; - return code; - -_err: - *ppColData = NULL; - return code; } -static int32_t tBlockDataAppendBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow) { +/* flag > 0: forward update + * flag == 0: insert + * flag < 0: backward update + */ +static int32_t tBlockDataUpsertBlockRow(SBlockData *pBlockData, SBlockData *pBlockDataFrom, int32_t iRow, + int32_t flag) { int32_t code = 0; SColVal cv = {0}; int32_t iColDataFrom = 0; - SColData *pColDataFrom = - (iColDataFrom < pBlockDataFrom->nColData) ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] : NULL; + SColData *pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; for (int32_t iColDataTo = 0; iColDataTo < pBlockData->nColData; iColDataTo++) { - SColData *pColDataTo = &((SColData *)pBlockData->aColData->pData)[iColDataTo]; + SColData *pColDataTo = &pBlockData->aColData[iColDataTo]; while (pColDataFrom && pColDataFrom->cid < pColDataTo->cid) { - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } if (pColDataFrom == NULL || pColDataFrom->cid > pColDataTo->cid) { - code = tColDataAppendValue(pColDataTo, &COL_VAL_NONE(pColDataTo->cid, pColDataTo->type)); - if (code) goto _exit; + cv = COL_VAL_NONE(pColDataTo->cid, pColDataTo->type); + if (flag == 0 && (code = tColDataAppendValue(pColDataTo, &cv))) goto _exit; } else { tColDataGetValue(pColDataFrom, iRow, &cv); - code = tColDataAppendValue(pColDataTo, &cv); - if (code) goto _exit; - - iColDataFrom++; - pColDataFrom = (iColDataFrom < pBlockDataFrom->nColData) - ? &((SColData *)pBlockDataFrom->aColData->pData)[iColDataFrom] - : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - - if (pRow->statis) { - TDRowValT vt = TD_VTYPE_MAX; - tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } + if (flag) { + code = tColDataUpdateValue(pColDataTo, &cv, flag > 0); } else { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; + code = tColDataAppendValue(pColDataTo, &cv); } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - col_id_t kvIter = 0; - col_id_t nKvCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - TDRowValT vt = TD_VTYPE_NONE; // default is NONE - SKvRowIdx *pKvIdx = NULL; - - while (kvIter < nKvCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); - if (pKvIdx->colId == pTColumn->colId) { - tdGetBitmapValTypeII(pBitmap, kvIter, &vt); - ++kvIter; - break; - } else if (pKvIdx->colId > pTColumn->colId) { - vt = TD_VTYPE_NONE; - break; - } else { - ++kvIter; - } - } - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pData, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + pColDataFrom = (++iColDataFrom < pBlockDataFrom->nColData) ? &pBlockDataFrom->aColData[iColDataFrom] : NULL; } } @@ -1480,159 +1330,75 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS if (pBlockData->uid == 0) { ASSERT(uid); code = tRealloc((uint8_t **)&pBlockData->aUid, sizeof(int64_t) * (pBlockData->nRow + 1)); - if (code) goto _err; + if (code) goto _exit; pBlockData->aUid[pBlockData->nRow] = uid; } // version code = tRealloc((uint8_t **)&pBlockData->aVersion, sizeof(int64_t) * (pBlockData->nRow + 1)); - if (code) goto _err; + if (code) goto _exit; pBlockData->aVersion[pBlockData->nRow] = TSDBROW_VERSION(pRow); // timestamp code = tRealloc((uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * (pBlockData->nRow + 1)); - if (code) goto _err; + if (code) goto _exit; pBlockData->aTSKEY[pBlockData->nRow] = TSDBROW_TS(pRow); - SColVal cv = {0}; - if (pRow->type == 0) { - if (TD_IS_TP_ROW(pRow->pTSRow)) { - code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else if (TD_IS_KV_ROW(pRow->pTSRow)) { - code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); - if (code) goto _err; - } else { - ASSERT(0); - } + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowUpsertColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData, 0 /* append */); + if (code) goto _exit; + } else if (pRow->type == TSDBROW_COL_FMT) { + code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, 0 /* append */); + if (code) goto _exit; } else { - code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); - if (code) goto _err; + ASSERT(0); } pBlockData->nRow++; - return code; - -_err: - return code; -} - -int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { - int32_t code = 0; - - int32_t iColData = 0; - for (int32_t iColDataFrom = 0; iColDataFrom < pBlockDataFrom->nColData; iColDataFrom++) { - SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); - - while (true) { - SColData *pColData; - if (iColData < pBlockData->nColData) { - pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - } else { - pColData = NULL; - } - - if (pColData == NULL || pColData->cid > pColDataFrom->cid) { - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; - - tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } - - iColData++; - break; - } else if (pColData->cid == pColDataFrom->cid) { - iColData++; - break; - } else { - iColData++; - } - } - } - _exit: return code; } - -int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { +static int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; - ASSERT(pBlockData->suid == pBlockData1->suid); - ASSERT(pBlockData->uid == pBlockData1->uid); - ASSERT(pBlockData1->nRow > 0); - ASSERT(pBlockData2->nRow > 0); - - tBlockDataClear(pBlockData); - - TSDBROW row1 = tsdbRowFromBlockData(pBlockData1, 0); - TSDBROW row2 = tsdbRowFromBlockData(pBlockData2, 0); - TSDBROW *pRow1 = &row1; - TSDBROW *pRow2 = &row2; - - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); - if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); - if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } + // version + int64_t lversion = pBlockData->aVersion[pBlockData->nRow - 1]; + int64_t rversion = TSDBROW_VERSION(pRow); + ASSERT(lversion != rversion); + if (rversion > lversion) { + pBlockData->aVersion[pBlockData->nRow - 1] = rversion; } - while (pRow1) { - code = tBlockDataAppendRow(pBlockData, pRow1, NULL, - pBlockData1->uid ? pBlockData1->uid : pBlockData1->aUid[pRow1->iRow]); + // update other rows + if (pRow->type == TSDBROW_ROW_FMT) { + code = tRowUpsertColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData, + (rversion > lversion) ? 1 : -1 /* update */); if (code) goto _exit; - - pRow1->iRow++; - if (pRow1->iRow < pBlockData1->nRow) { - *pRow1 = tsdbRowFromBlockData(pBlockData1, pRow1->iRow); - } else { - pRow1 = NULL; - } - } - - while (pRow2) { - code = tBlockDataAppendRow(pBlockData, pRow2, NULL, - pBlockData2->uid ? pBlockData2->uid : pBlockData2->aUid[pRow2->iRow]); + } else if (pRow->type == TSDBROW_COL_FMT) { + code = tBlockDataUpsertBlockRow(pBlockData, pRow->pBlockData, pRow->iRow, (rversion > lversion) ? 1 : -1); if (code) goto _exit; - - pRow2->iRow++; - if (pRow2->iRow < pBlockData2->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockData2, pRow2->iRow); - } else { - pRow2 = NULL; - } + } else { + ASSERT(0); } _exit: return code; } -SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx) { - ASSERT(idx >= 0 && idx < pBlockData->nColData); - return (SColData *)taosArrayGet(pBlockData->aColData, idx); +int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid) { + if (pBlockData->nRow == 0) { + return 1; + } else if (pBlockData->aTSKEY[pBlockData->nRow - 1] == TSDBROW_TS(pRow)) { + return pBlockData->nRow; + } else { + return pBlockData->nRow + 1; + } +} + +int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { + if (pBlockData->nRow > 0 && pBlockData->aTSKEY[pBlockData->nRow - 1] == TSDBROW_TS(pRow)) { + return tBlockDataUpdateRow(pBlockData, pRow, pTSchema); + } else { + return tBlockDataAppendRow(pBlockData, pRow, pTSchema, uid); + } } void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { @@ -1641,7 +1407,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD int32_t ridx = pBlockData->nColData - 1; while (lidx <= ridx) { - int32_t midx = (lidx + ridx) / 2; + int32_t midx = (lidx + ridx) >> 1; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); @@ -1788,15 +1554,25 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; + int32_t nColData = 0; int32_t nt = 0; while (nt < hdr.szBlkCol) { SBlockCol blockCol = {0}; nt += tGetBlockCol(pIn + n + nt, &blockCol); - ASSERT(nt <= hdr.szBlkCol); + ++nColData; + } + ASSERT(nt == hdr.szBlkCol); - SColData *pColData; - code = tBlockDataAddColData(pBlockData, &pColData); - if (code) goto _exit; + code = tBlockDataAdjustColData(pBlockData, nColData); + if (code) goto _exit; + + nt = 0; + int32_t iColData = 0; + while (nt < hdr.szBlkCol) { + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); + + SColData *pColData = &pBlockData->aColData[iColData++]; tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); if (blockCol.flag == HAS_NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 49d5eaac43c6dbd2356bf95ca4cfb9c70cb350c7..bd2d26380424852f0edb709f75dc828705980345 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -22,17 +22,22 @@ * us: 3600*1000000*8765*1000 // 1970 + 1000 years * ns: 3600*1000000000*8765*292 // 1970 + 292 years */ -static int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L}; +int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L}; // static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); -int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *pRsp) { - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - int32_t affectedrows = 0; - int32_t numOfRows = 0; +int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) { + int32_t arrSize = 0; + int32_t affectedrows = 0; + int32_t numOfRows = 0; - ASSERT(pTsdb->mem != NULL); + if (ASSERTS(pTsdb->mem != NULL, "vgId:%d, mem is NULL", TD_VID(pTsdb->pVnode))) { + return -1; + } + + if (pMsg) { + arrSize = taosArrayGetSize(pMsg->aSubmitTbData); + } // scan and convert if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { @@ -43,18 +48,10 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * } // loop to insert - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { - return -1; - } - while (true) { - SSubmitBlkRsp r = {0}; - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { + for (int32_t i = 0; i < arrSize; ++i) { + if ((terrno = tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows)) < 0) { return -1; } - - numOfRows += msgIter.numOfRows; } if (pRsp != NULL) { @@ -65,104 +62,51 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * return 0; } -#if 0 -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, STable *pTable, STSRow *row, TSKEY minKey, TSKEY maxKey, - TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); - if (rowKey < minKey || rowKey > maxKey) { - tsdbError("vgId:%d, table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 - " maxKey %" PRId64 " row key %" PRId64, - REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey, - rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; - } - - return 0; -} -#endif - -static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, STSRow *row, TSKEY minKey, TSKEY maxKey, +static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey, TSKEY now) { - TSKEY rowKey = TD_ROW_KEY(row); if (rowKey < minKey || rowKey > maxKey) { tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 " maxKey %" PRId64 " row key %" PRId64, TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey); - terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - return -1; + return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; } return 0; } -int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { - ASSERT(pMsg != NULL); - // STsdbMeta * pMeta = pTsdb->tsdbMeta; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SSubmitBlkIter blkIter = {0}; - STSRow *row = NULL; - STsdbKeepCfg *pCfg = &pTsdb->keepCfg; - TSKEY now = taosGetTimestamp(pCfg->precision); - TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; - TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; - - terrno = TSDB_CODE_SUCCESS; - // pMsg->length = htonl(pMsg->length); - // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; - - // pBlock->uid = htobe64(pBlock->uid); - // pBlock->suid = htobe64(pBlock->suid); - // pBlock->sversion = htonl(pBlock->sversion); - // pBlock->dataLen = htonl(pBlock->dataLen); - // pBlock->schemaLen = htonl(pBlock->schemaLen); - // pBlock->numOfRows = htonl(pBlock->numOfRows); - -#if 0 - if (pBlock->tid <= 0 || pBlock->tid >= pMeta->maxTables) { - tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - STable *pTable = pMeta->tables[pBlock->tid]; - if (pTable == NULL || TABLE_UID(pTable) != pBlock->uid) { - tsdbError("vgId:%d, failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pTsdb), pBlock->uid, - pBlock->tid); - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - return -1; - } - - if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tsdbError("vgId:%d, invalid action trying to insert a super table %s", REPO_ID(pTsdb), TABLE_CHAR_NAME(pTable)); - terrno = TSDB_CODE_TDB_INVALID_ACTION; - return -1; - } - - // Check schema version and update schema if needed - if (tsdbCheckTableSchema(pTsdb, pBlock, pTable) < 0) { - if (terrno == TSDB_CODE_TDB_TABLE_RECONFIGURE) { - continue; - } else { - return -1; +int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { + int32_t code = 0; + STsdbKeepCfg *pCfg = &pTsdb->keepCfg; + TSKEY now = taosGetTimestamp(pCfg->precision); + TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; + int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); + + for (int32_t i = 0; i < size; ++i) { + SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); + if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData = TARRAY_SIZE(pData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol); + if (nColData > 0) { + int32_t nRows = aColData[0].nVal; + TSKEY *aKey = (TSKEY *)aColData[0].pData; + for (int32_t r = 0; r < nRows; ++r) { + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now)) < 0) { + goto _exit; + } + } } - } -#endif - tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); - while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { - if (tsdbCheckRowRange(pTsdb, msgIter.uid, row, minKey, maxKey, now) < 0) { - return -1; + } else { + int32_t nRows = taosArrayGetSize(pData->aRowP); + for (int32_t r = 0; r < nRows; ++r) { + SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r); + if ((code = tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now)) < 0) { + goto _exit; + } } } } - if (terrno != TSDB_CODE_SUCCESS) return -1; - return 0; +_exit: + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeBufPool.c b/source/dnode/vnode/src/vnd/vnodeBufPool.c index 88abc1b3f0e25537707ba44b20860b77c55b35df..40112c55791fe8d2cd3545f57169e2ba7ec33838 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufPool.c @@ -16,9 +16,7 @@ #include "vnd.h" /* ------------------------ STRUCTURES ------------------------ */ -#define VNODE_BUFPOOL_SEGMENTS 3 - -static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) { +static int vnodeBufPoolCreate(SVnode *pVnode, int32_t id, int64_t size, SVBufPool **ppPool) { SVBufPool *pPool; pPool = taosMemoryMalloc(sizeof(SVBufPool) + size); @@ -26,6 +24,21 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + memset(pPool, 0, sizeof(SVBufPool)); + + // query handle list + taosThreadMutexInit(&pPool->mutex, NULL); + pPool->nQuery = 0; + pPool->qList.pNext = &pPool->qList; + pPool->qList.ppNext = &pPool->qList.pNext; + + pPool->pVnode = pVnode; + pPool->id = id; + pPool->ptr = pPool->node.data; + pPool->pTail = &pPool->node; + pPool->node.prev = NULL; + pPool->node.pnext = &pPool->pTail; + pPool->node.size = size; if (VND_IS_RSMA(pVnode)) { pPool->lock = taosMemoryMalloc(sizeof(TdThreadSpinlock)); @@ -44,16 +57,6 @@ static int vnodeBufPoolCreate(SVnode *pVnode, int64_t size, SVBufPool **ppPool) pPool->lock = NULL; } - pPool->next = NULL; - pPool->pVnode = pVnode; - pPool->nRef = 0; - pPool->size = 0; - pPool->ptr = pPool->node.data; - pPool->pTail = &pPool->node; - pPool->node.prev = NULL; - pPool->node.pnext = &pPool->pTail; - pPool->node.size = size; - *ppPool = pPool; return 0; } @@ -64,27 +67,25 @@ static int vnodeBufPoolDestroy(SVBufPool *pPool) { taosThreadSpinDestroy(pPool->lock); taosMemoryFree((void *)pPool->lock); } + taosThreadMutexDestroy(&pPool->mutex); taosMemoryFree(pPool); return 0; } int vnodeOpenBufPool(SVnode *pVnode) { - SVBufPool *pPool = NULL; - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - - ASSERT(pVnode->pPool == NULL); + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - for (int i = 0; i < 3; i++) { + for (int i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { // create pool - if (vnodeBufPoolCreate(pVnode, size, &pPool)) { + if (vnodeBufPoolCreate(pVnode, i, size, &pVnode->aBufPool[i])) { vError("vgId:%d, failed to open vnode buffer pool since %s", TD_VID(pVnode), tstrerror(terrno)); vnodeCloseBufPool(pVnode); return -1; } - // add pool to vnode - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; + // add to free list + pVnode->aBufPool[i]->freeNext = pVnode->freeList; + pVnode->freeList = pVnode->aBufPool[i]; } vDebug("vgId:%d, vnode buffer pool is opened, size:%" PRId64, TD_VID(pVnode), size); @@ -92,25 +93,19 @@ int vnodeOpenBufPool(SVnode *pVnode) { } int vnodeCloseBufPool(SVnode *pVnode) { - SVBufPool *pPool; - - taosThreadMutexLock(&pVnode->mutex); - for (pPool = pVnode->pPool; pPool; pPool = pVnode->pPool) { - pVnode->pPool = pPool->next; - vnodeBufPoolDestroy(pPool); - } - - if (pVnode->inUse) { - vnodeBufPoolDestroy(pVnode->inUse); - pVnode->inUse = NULL; + for (int32_t i = 0; i < VNODE_BUFPOOL_SEGMENTS; i++) { + if (pVnode->aBufPool[i]) { + vnodeBufPoolDestroy(pVnode->aBufPool[i]); + pVnode->aBufPool[i] = NULL; + } } - taosThreadMutexUnlock(&pVnode->mutex); vDebug("vgId:%d, vnode buffer pool is closed", TD_VID(pVnode)); return 0; } void vnodeBufPoolReset(SVBufPool *pPool) { + ASSERT(pPool->nQuery == 0); for (SVBufPoolNode *pNode = pPool->pTail; pNode->prev; pNode = pPool->pTail) { ASSERT(pNode->pnext == &pPool->pTail); pNode->prev->pnext = &pPool->pTail; @@ -217,38 +212,119 @@ void vnodeBufPoolRef(SVBufPool *pPool) { ASSERT(nRef > 0); } -void vnodeBufPoolUnRef(SVBufPool *pPool) { - if (pPool == NULL) { - return; +void vnodeBufPoolAddToFreeList(SVBufPool *pPool) { + SVnode *pVnode = pPool->pVnode; + + int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; + if (pPool->node.size != size) { + SVBufPool *pNewPool = NULL; + if (vnodeBufPoolCreate(pVnode, pPool->id, size, &pNewPool) < 0) { + vWarn("vgId:%d, failed to change buffer pool of id %d size from %" PRId64 " to %" PRId64 " since %s", + TD_VID(pVnode), pPool->id, pPool->node.size, size, tstrerror(errno)); + } else { + vInfo("vgId:%d, buffer pool of id %d size changed from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->id, + pPool->node.size, size); + + vnodeBufPoolDestroy(pPool); + pPool = pNewPool; + pVnode->aBufPool[pPool->id] = pPool; + } } - int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); - if (nRef == 0) { - SVnode *pVnode = pPool->pVnode; - - vnodeBufPoolReset(pPool); - - taosThreadMutexLock(&pVnode->mutex); - - int64_t size = pVnode->config.szBuf / VNODE_BUFPOOL_SEGMENTS; - if (pPool->node.size != size) { - SVBufPool *pPoolT = NULL; - if (vnodeBufPoolCreate(pVnode, size, &pPoolT) < 0) { - vWarn("vgId:%d, try to change buf pools size from %" PRId64 " to %" PRId64 " since %s", TD_VID(pVnode), - pPool->node.size, size, tstrerror(errno)); - } else { - vnodeBufPoolDestroy(pPool); - pPool = pPoolT; - vDebug("vgId:%d, change buf pools size from %" PRId64 " to %" PRId64, TD_VID(pVnode), pPool->node.size, size); - } + + // add to free list + vDebug("vgId:%d, buffer pool %p of id %d is added to free list", TD_VID(pVnode), pPool, pPool->id); + vnodeBufPoolReset(pPool); + pPool->freeNext = pVnode->freeList; + pVnode->freeList = pPool; + taosThreadCondSignal(&pVnode->poolNotEmpty); +} + +void vnodeBufPoolUnRef(SVBufPool *pPool, bool proactive) { + if (pPool == NULL) return; + + SVnode *pVnode = pPool->pVnode; + + if (proactive) taosThreadMutexLock(&pVnode->mutex); + + if (atomic_sub_fetch_32(&pPool->nRef, 1) > 0) goto _exit; + + // remove from recycle queue or on-recycle position + if (pVnode->onRecycle == pPool) { + pVnode->onRecycle = NULL; + } else { + if (pPool->recyclePrev) { + pPool->recyclePrev->recycleNext = pPool->recycleNext; + } else { + pVnode->recycleHead = pPool->recycleNext; + } + + if (pPool->recycleNext) { + pPool->recycleNext->recyclePrev = pPool->recyclePrev; + } else { + pVnode->recycleTail = pPool->recyclePrev; } + pPool->recyclePrev = pPool->recycleNext = NULL; + } + + vnodeBufPoolAddToFreeList(pPool); + +_exit: + if (proactive) taosThreadMutexUnlock(&pVnode->mutex); + return; +} + +int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) { + int32_t code = 0; + + taosThreadMutexLock(&pPool->mutex); - pPool->next = pVnode->pPool; - pVnode->pPool = pPool; - taosThreadCondSignal(&pVnode->poolNotEmpty); + pQNode->pNext = pPool->qList.pNext; + pQNode->ppNext = &pPool->qList.pNext; + pPool->qList.pNext->ppNext = &pQNode->pNext; + pPool->qList.pNext = pQNode; + pPool->nQuery++; - if (pVnode->inUse == pPool) { - pVnode->inUse = NULL; + taosThreadMutexUnlock(&pPool->mutex); + +_exit: + return code; +} + +void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) { + int32_t code = 0; + + if (proactive) taosThreadMutexLock(&pPool->mutex); + + pQNode->pNext->ppNext = pQNode->ppNext; + *pQNode->ppNext = pQNode->pNext; + pPool->nQuery--; + + if (proactive) taosThreadMutexUnlock(&pPool->mutex); +} + +int32_t vnodeBufPoolRecycle(SVBufPool *pPool) { + int32_t code = 0; + + SVnode *pVnode = pPool->pVnode; + + vDebug("vgId:%d, recycle buffer pool %p of id %d", TD_VID(pVnode), pPool, pPool->id); + + taosThreadMutexLock(&pPool->mutex); + + SQueryNode *pNode = pPool->qList.pNext; + while (pNode != &pPool->qList) { + SQueryNode *pTNode = pNode->pNext; + + int32_t rc = pNode->reseek(pNode->pQHandle); + if (rc == 0 || rc == TSDB_CODE_VND_QUERY_BUSY) { + pNode = pTNode; + } else { + code = rc; + goto _exit; } - taosThreadMutexUnlock(&pVnode->mutex); } + +_exit: + taosThreadMutexUnlock(&pPool->mutex); + return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 69af536a4454909be7816ebda880e99e648656c3..c326c8bfacbd897fc15527edaef5cd8e77c40f5f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -262,9 +262,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code); if (code < 0) return -1; tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn); - if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code); + if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); + if (code < 0) return -1; vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index e8280ea7518012e887d80004b547b9c59ca01f04..0779388ba9ca6da06ff0bfe3ce65219904118f0e 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -21,63 +21,126 @@ static int vnodeEncodeInfo(const SVnodeInfo *pInfo, char **ppData); static int vnodeCommitImpl(SCommitInfo *pInfo); -#define WAIT_TIME_MILI_SEC 50 +#define WAIT_TIME_MILI_SEC 10 // miliseconds -int vnodeBegin(SVnode *pVnode) { - // alloc buffer pool - int32_t nTry = 0; - - taosThreadMutexLock(&pVnode->mutex); - - while (pVnode->pPool == NULL) { - vInfo("vgId:%d no free buffer pool on %d try, wait %d ms...", TD_VID(pVnode), ++nTry, WAIT_TIME_MILI_SEC); +static int32_t vnodeTryRecycleBufPool(SVnode *pVnode) { + int32_t code = 0; - struct timeval tv; - struct timespec ts; - taosGetTimeOfDay(&tv); - ts.tv_nsec = tv.tv_usec * 1000 + WAIT_TIME_MILI_SEC * 1000000; - if (ts.tv_nsec > 999999999l) { - ts.tv_sec = tv.tv_sec + 1; - ts.tv_nsec -= 1000000000l; + if (pVnode->onRecycle == NULL) { + if (pVnode->recycleHead == NULL) { + vDebug("vgId:%d, no recyclable buffer pool", TD_VID(pVnode)); + goto _exit; } else { - ts.tv_sec = tv.tv_sec; + vDebug("vgId:%d, buffer pool %p of id %d on recycle queue, try to recycle", TD_VID(pVnode), pVnode->recycleHead, + pVnode->recycleHead->id); + + pVnode->onRecycle = pVnode->recycleHead; + if (pVnode->recycleHead == pVnode->recycleTail) { + pVnode->recycleHead = pVnode->recycleTail = NULL; + } else { + pVnode->recycleHead = pVnode->recycleHead->recycleNext; + pVnode->recycleHead->recyclePrev = NULL; + } + pVnode->onRecycle->recycleNext = pVnode->onRecycle->recyclePrev = NULL; } + } - int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); - if (rc && rc != ETIMEDOUT) { - terrno = TAOS_SYSTEM_ERROR(rc); - taosThreadMutexUnlock(&pVnode->mutex); - return -1; - } + code = vnodeBufPoolRecycle(pVnode->onRecycle); + if (code) goto _exit; + +_exit: + if (code) { + vError("vgId:%d, %s failed since %s", TD_VID(pVnode), __func__, tstrerror(code)); } + return code; +} +static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; - pVnode->inUse = pVnode->pPool; - pVnode->inUse->nRef = 1; - pVnode->pPool = pVnode->inUse->next; - pVnode->inUse->next = NULL; + taosThreadMutexLock(&pVnode->mutex); + int32_t nTry = 0; + for (;;) { + ++nTry; + + if (pVnode->freeList) { + vDebug("vgId:%d, allocate free buffer pool on %d try, pPool:%p id:%d", TD_VID(pVnode), nTry, pVnode->freeList, + pVnode->freeList->id); + + pVnode->inUse = pVnode->freeList; + pVnode->inUse->nRef = 1; + pVnode->freeList = pVnode->inUse->freeNext; + pVnode->inUse->freeNext = NULL; + break; + } else { + vDebug("vgId:%d, no free buffer pool on %d try, try to recycle...", TD_VID(pVnode), nTry); + + code = vnodeTryRecycleBufPool(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pVnode->freeList == NULL) { + vDebug("vgId:%d, no free buffer pool on %d try, wait %d ms...", TD_VID(pVnode), nTry, WAIT_TIME_MILI_SEC); + + struct timeval tv; + struct timespec ts; + taosGetTimeOfDay(&tv); + ts.tv_nsec = tv.tv_usec * 1000 + WAIT_TIME_MILI_SEC * 1000000; + if (ts.tv_nsec > 999999999l) { + ts.tv_sec = tv.tv_sec + 1; + ts.tv_nsec -= 1000000000l; + } else { + ts.tv_sec = tv.tv_sec; + } + + int32_t rc = taosThreadCondTimedWait(&pVnode->poolNotEmpty, &pVnode->mutex, &ts); + if (rc && rc != ETIMEDOUT) { + code = TAOS_SYSTEM_ERROR(rc); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } + } + +_exit: taosThreadMutexUnlock(&pVnode->mutex); + if (code) { + vError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; +} +int vnodeBegin(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; + + // alloc buffer pool + code = vnodeGetBufPoolToUse(pVnode); + TSDB_CHECK_CODE(code, lino, _exit); - pVnode->state.commitID++; // begin meta if (metaBegin(pVnode->pMeta, META_BEGIN_HEAP_BUFFERPOOL) < 0) { - vError("vgId:%d, failed to begin meta since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin tsdb if (tsdbBegin(pVnode->pTsdb) < 0) { - vError("vgId:%d, failed to begin tsdb since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } // begin sma if (VND_IS_RSMA(pVnode) && smaBegin(pVnode->pSma) < 0) { - vError("vgId:%d, failed to begin sma since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; + code = terrno; + TSDB_CHECK_CODE(code, lino, _exit); } - return 0; +_exit: + if (code) { + terrno = code; + vError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } + return code; } void vnodeUpdCommitSched(SVnode *pVnode) { @@ -155,7 +218,7 @@ _err: return -1; } -int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) { +int vnodeCommitInfo(const char *dir) { char fname[TSDB_FILENAME_LEN]; char tfname[TSDB_FILENAME_LEN]; @@ -167,8 +230,7 @@ int vnodeCommitInfo(const char *dir, const SVnodeInfo *pInfo) { return -1; } - vInfo("vgId:%d, vnode info is committed", pInfo->config.vgId); - + vInfo("vnode info is committed, dir:%s", dir); return 0; } @@ -235,7 +297,7 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { pInfo->info.config = pVnode->config; pInfo->info.state.committed = pVnode->state.applied; pInfo->info.state.commitTerm = pVnode->state.applyTerm; - pInfo->info.state.commitID = pVnode->state.commitID; + pInfo->info.state.commitID = ++pVnode->state.commitID; pInfo->pVnode = pVnode; pInfo->txn = metaGetTxn(pVnode->pMeta); @@ -253,35 +315,70 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) { } tsdbPrepareCommit(pVnode->pTsdb); - smaPrepareAsyncCommit(pVnode->pSma); metaPrepareAsyncCommit(pVnode->pMeta); - vnodeBufPoolUnRef(pVnode->inUse); + code = smaPrepareAsyncCommit(pVnode->pSma); + if (code) goto _exit; + + taosThreadMutexLock(&pVnode->mutex); + ASSERT(pVnode->onCommit == NULL); + pVnode->onCommit = pVnode->inUse; + pVnode->inUse = NULL; + taosThreadMutexUnlock(&pVnode->mutex); _exit: if (code) { vError("vgId:%d, %s failed at line %d since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code), pVnode->state.commitID); } else { - vDebug("vgId:%d, %s done", TD_VID(pVnode), __func__); + vDebug("vgId:%d, %s done, commit id:%" PRId64, TD_VID(pVnode), __func__, pInfo->info.state.commitID); } + return code; } +static void vnodeReturnBufPool(SVnode *pVnode) { + taosThreadMutexLock(&pVnode->mutex); + SVBufPool *pPool = pVnode->onCommit; + int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1); + + pVnode->onCommit = NULL; + if (nRef == 0) { + vnodeBufPoolAddToFreeList(pPool); + } else if (nRef > 0) { + vDebug("vgId:%d, buffer pool %p of id %d is added to recycle queue", TD_VID(pVnode), pPool, pPool->id); + + if (pVnode->recycleTail == NULL) { + pPool->recyclePrev = pPool->recycleNext = NULL; + pVnode->recycleHead = pVnode->recycleTail = pPool; + } else { + pPool->recyclePrev = pVnode->recycleTail; + pPool->recycleNext = NULL; + pVnode->recycleTail->recycleNext = pPool; + pVnode->recycleTail = pPool; + } + } else { + ASSERT(0); + } + + taosThreadMutexUnlock(&pVnode->mutex); +} static int32_t vnodeCommitTask(void *arg) { int32_t code = 0; SCommitInfo *pInfo = (SCommitInfo *)arg; + SVnode *pVnode = pInfo->pVnode; // commit code = vnodeCommitImpl(pInfo); if (code) goto _exit; - // end commit - tsem_post(&pInfo->pVnode->canCommit); + vnodeReturnBufPool(pVnode); _exit: + // end commit + tsem_post(&pVnode->canCommit); taosMemoryFree(pInfo); return code; } @@ -302,14 +399,15 @@ int vnodeAsyncCommit(SVnode *pVnode) { } // schedule the task - vnodeScheduleTask(vnodeCommitTask, pInfo); + code = vnodeScheduleTask(vnodeCommitTask, pInfo); _exit: if (code) { if (NULL != pInfo) { taosMemoryFree(pInfo); } - vError("vgId:%d, vnode async commit failed since %s, commitId:%" PRId64, TD_VID(pVnode), tstrerror(code), + tsem_post(&pVnode->canCommit); + vError("vgId:%d, %s failed since %s, commit id:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), pVnode->state.commitID); } else { vInfo("vgId:%d, vnode async commit done, commitId:%" PRId64 " term:%" PRId64 " applied:%" PRId64, TD_VID(pVnode), @@ -366,7 +464,7 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { } // commit info - if (vnodeCommitInfo(dir, &pInfo->info) < 0) { + if (vnodeCommitInfo(dir) < 0) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } diff --git a/source/dnode/vnode/src/vnd/vnodeCompact.c b/source/dnode/vnode/src/vnd/vnodeCompact.c new file mode 100644 index 0000000000000000000000000000000000000000..16e39d75dce38d40940df3b4ebf886f972f2caaf --- /dev/null +++ b/source/dnode/vnode/src/vnd/vnodeCompact.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "vnd.h" + +static int32_t vnodeCompactTask(void *param) { + int32_t code = 0; + int32_t lino = 0; + + SCompactInfo *pInfo = (SCompactInfo *)param; + SVnode *pVnode = pInfo->pVnode; + + // do compact + code = tsdbCompact(pInfo->pVnode->pTsdb, pInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + // end compact + char dir[TSDB_FILENAME_LEN] = {0}; + if (pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); + } + vnodeCommitInfo(dir); + +_exit: + tsem_post(&pInfo->pVnode->canCommit); + taosMemoryFree(pInfo); + return code; +} +static int32_t vnodePrepareCompact(SVnode *pVnode, SCompactInfo *pInfo) { + int32_t code = 0; + int32_t lino = 0; + + tsem_wait(&pVnode->canCommit); + + pInfo->pVnode = pVnode; + pInfo->flag = 0; + pInfo->commitID = ++pVnode->state.commitID; + + char dir[TSDB_FILENAME_LEN] = {0}; + SVnodeInfo info = {0}; + + if (pVnode->pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path); + } + + if (vnodeLoadInfo(dir, &info) < 0) { + code = terrno; + goto _exit; + } + + info.state.commitID = pInfo->commitID; + + if (vnodeSaveInfo(dir, &info) < 0) { + code = terrno; + goto _exit; + } + +_exit: + if (code) { + vError("vgId:%d %s failed at line %d since %s, commit ID:%" PRId64, TD_VID(pVnode), __func__, lino, tstrerror(code), + pVnode->state.commitID); + } else { + vDebug("vgId:%d %s done, commit ID:%" PRId64, TD_VID(pVnode), __func__, pVnode->state.commitID); + } + return code; +} +int32_t vnodeAsyncCompact(SVnode *pVnode) { + int32_t code = 0; + int32_t lino = 0; + + SCompactInfo *pInfo = taosMemoryCalloc(1, sizeof(*pInfo)); + if (pInfo == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + vnodeAsyncCommit(pVnode); + + code = vnodePrepareCompact(pVnode, pInfo); + TSDB_CHECK_CODE(code, lino, _exit); + + vnodeScheduleTask(vnodeCompactTask, pInfo); + +_exit: + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + if (pInfo) taosMemoryFree(pInfo); + } else { + vInfo("vgId:%d %s done", TD_VID(pVnode), __func__); + } + return code; +} + +int32_t vnodeSyncCompact(SVnode *pVnode) { + vnodeAsyncCompact(pVnode); + tsem_wait(&pVnode->canCommit); + tsem_post(&pVnode->canCommit); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 97ee2e4a89597aa6021b08ee35bc1dd27684a353..c7d155be0d50600e419940b03297d2bad6d29528 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -48,8 +48,8 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { info.state.applied = -1; info.state.commitID = 0; - vInfo("vgId:%d, save config while create", pCfg->vgId); - if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) { + vInfo("vgId:%d, save config while create", info.config.vgId); + if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir) < 0) { vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno)); return -1; } @@ -58,7 +58,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { return 0; } -int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { +int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { SVnodeInfo info = {0}; char dir[TSDB_FILENAME_LEN] = {0}; int32_t ret = 0; @@ -86,7 +86,7 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); - tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } @@ -97,7 +97,7 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { return -1; } - ret = vnodeCommitInfo(dir, &info); + ret = vnodeCommitInfo(dir); if (ret < 0) { vError("vgId:%d, failed to commit vnode config since %s", pReq->vgId, tstrerror(terrno)); return -1; @@ -107,6 +107,117 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) { return 0; } +int32_t vnodeRenameVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, STfs *pTfs) { + int32_t ret = tfsRename(pTfs, srcPath, dstPath); + if (ret != 0) return ret; + + char oldRname[TSDB_FILENAME_LEN] = {0}; + char newRname[TSDB_FILENAME_LEN] = {0}; + char tsdbPath[TSDB_FILENAME_LEN] = {0}; + char tsdbFilePrefix[TSDB_FILENAME_LEN] = {0}; + snprintf(tsdbPath, TSDB_FILENAME_LEN, "%s%stsdb", dstPath, TD_DIRSEP); + snprintf(tsdbFilePrefix, TSDB_FILENAME_LEN, "tsdb%sv", TD_DIRSEP); + + STfsDir *tsdbDir = tfsOpendir(pTfs, tsdbPath); + if (tsdbDir == NULL) return 0; + + while (1) { + const STfsFile *tsdbFile = tfsReaddir(tsdbDir); + if (tsdbFile == NULL) break; + if (tsdbFile->rname[0] == '\0') continue; + tstrncpy(oldRname, tsdbFile->rname, TSDB_FILENAME_LEN); + + char *tsdbFilePrefixPos = strstr(oldRname, tsdbFilePrefix); + if (tsdbFilePrefixPos == NULL) continue; + + int32_t tsdbFileVgId = atoi(tsdbFilePrefixPos + 6); + if (tsdbFileVgId == srcVgId) { + char *tsdbFileSurfixPos = strstr(tsdbFilePrefixPos, "f"); + if (tsdbFileSurfixPos == NULL) continue; + + tsdbFilePrefixPos[6] = 0; + snprintf(newRname, TSDB_FILENAME_LEN, "%s%d%s", oldRname, dstVgId, tsdbFileSurfixPos); + vInfo("vgId:%d, rename file from %s to %s", dstVgId, tsdbFile->rname, newRname); + + ret = tfsRename(pTfs, tsdbFile->rname, newRname); + if (ret != 0) { + vInfo("vgId:%d, failed to rename file from %s to %s since %s", dstVgId, tsdbFile->rname, newRname, terrstr()); + tfsClosedir(tsdbDir); + return ret; + } + } + } + + tfsClosedir(tsdbDir); + return 0; +} + +int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnodeHashRangeReq *pReq, STfs *pTfs) { + SVnodeInfo info = {0}; + char dir[TSDB_FILENAME_LEN] = {0}; + int32_t ret = 0; + + if (pTfs) { + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, srcPath); + } else { + snprintf(dir, TSDB_FILENAME_LEN, "%s", srcPath); + } + + // todo add stat file to handle exception while vnode open + + ret = vnodeLoadInfo(dir, &info); + if (ret < 0) { + vError("vgId:%d, failed to read vnode config from %s since %s", pReq->srcVgId, srcPath, tstrerror(terrno)); + return -1; + } + + vInfo("vgId:%d, alter hashrange from [%u, %u] to [%u, %u]", pReq->srcVgId, info.config.hashBegin, info.config.hashEnd, + pReq->hashBegin, pReq->hashEnd); + info.config.vgId = pReq->dstVgId; + info.config.hashBegin = pReq->hashBegin; + info.config.hashEnd = pReq->hashEnd; + info.config.walCfg.vgId = pReq->dstVgId; + + SSyncCfg *pCfg = &info.config.syncCfg; + pCfg->myIndex = 0; + pCfg->replicaNum = 1; + memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); + + vInfo("vgId:%d, alter vnode replicas to 1", pReq->srcVgId); + SNodeInfo *pNode = &pCfg->nodeInfo[0]; + pNode->nodePort = tsServerPort; + tstrncpy(pNode->nodeFqdn, tsLocalFqdn, TSDB_FQDN_LEN); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + vInfo("vgId:%d, ep:%s:%u dnode:%d", pReq->srcVgId, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + + info.config.syncCfg = *pCfg; + + ret = vnodeSaveInfo(dir, &info); + if (ret < 0) { + vError("vgId:%d, failed to save vnode config since %s", pReq->dstVgId, tstrerror(terrno)); + return -1; + } + + ret = vnodeCommitInfo(dir); + if (ret < 0) { + vError("vgId:%d, failed to commit vnode config since %s", pReq->dstVgId, tstrerror(terrno)); + return -1; + } + + vInfo("vgId:%d, rename %s to %s", pReq->dstVgId, srcPath, dstPath); + ret = vnodeRenameVgroupId(srcPath, dstPath, pReq->srcVgId, pReq->dstVgId, pTfs); + if (ret < 0) { + vError("vgId:%d, failed to rename vnode from %s to %s since %s", pReq->dstVgId, srcPath, dstPath, + tstrerror(terrno)); + return -1; + } + + // todo vnode compact here + + vInfo("vgId:%d, vnode hashrange is altered", info.config.vgId); + return 0; +} + void vnodeDestroy(const char *path, STfs *pTfs) { vInfo("path:%s is removed while destroy vnode", path); tfsRmdir(pTfs, path); @@ -134,6 +245,21 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { return NULL; } + // save vnode info on dnode ep changed + bool updated = false; + SSyncCfg *pCfg = &info.config.syncCfg; + for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { + updated = true; + } + } + if (updated) { + vInfo("vgId:%d, save vnode info since dnode info changed", info.config.vgId); + (void)vnodeSaveInfo(dir, &info); + (void)vnodeCommitInfo(dir); + } + // create handle pVnode = taosMemoryCalloc(1, sizeof(*pVnode) + strlen(path) + 1); if (pVnode == NULL) { @@ -240,7 +366,7 @@ _err: if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pMeta) metaClose(&pVnode->pMeta); - if (pVnode->pPool) vnodeCloseBufPool(pVnode); + if (pVnode->freeList) vnodeCloseBufPool(pVnode); tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 1199127f6d42bf2d3f7488fb120e8fa9ab09c490..c017266839a872711b5262ee6671d17d3a3a76b6 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -15,13 +15,13 @@ #include "vnd.h" -#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ - do { \ - int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ - ASSERT(newVal >= 0); \ - if (newVal < 0) { \ - vWarn("vgId:%d %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \ - } \ +#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \ + do { \ + int##vType##_t newVal = atomic_sub_fetch_##vType(&(pVar), (oVal)); \ + ASSERT(newVal >= 0); \ + if (newVal < 0) { \ + vWarn("vgId:%d, %s, abnormal val:%" PRIi64 ", old val:%" PRIi64, TD_VID(pVnode), tags, newVal, (oVal)); \ + } \ } while (0) int vnodeQueryOpen(SVnode *pVnode) { @@ -96,6 +96,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.numOfColumns = schema.nCols; metaRsp.precision = pVnode->config.tsdbCfg.precision; metaRsp.sversion = schema.version; + metaRsp.tversion = schemaTag.version; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); @@ -196,7 +197,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; cfgRsp.commentLen = mer1.me.ctbEntry.commentLen; if (mer1.me.ctbEntry.commentLen > 0) { - cfgRsp.pComment = strdup(mer1.me.ctbEntry.comment); + cfgRsp.pComment = taosStrdup(mer1.me.ctbEntry.comment); } STag *pTag = (STag *)mer1.me.ctbEntry.pTags; cfgRsp.tagsLen = pTag->len; @@ -207,7 +208,7 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { cfgRsp.ttl = mer1.me.ntbEntry.ttlDays; cfgRsp.commentLen = mer1.me.ntbEntry.commentLen; if (mer1.me.ntbEntry.commentLen > 0) { - cfgRsp.pComment = strdup(mer1.me.ntbEntry.comment); + cfgRsp.pComment = taosStrdup(mer1.me.ntbEntry.comment); } } else { ASSERT(0); @@ -264,26 +265,25 @@ _exit: return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void vnodeFreeSBatchRspMsg(void* p) { +static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) { if (NULL == p) { return; } - SBatchRspMsg* pRsp = (SBatchRspMsg*)p; + SBatchRspMsg *pRsp = (SBatchRspMsg *)p; rpcFreeCont(pRsp->msg); } - int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { - int32_t code = 0; - int32_t rspSize = 0; - SBatchReq batchReq = {0}; - SBatchMsg *req = NULL; + int32_t code = 0; + int32_t rspSize = 0; + SBatchReq batchReq = {0}; + SBatchMsg *req = NULL; SBatchRspMsg rsp = {0}; - SBatchRsp batchRsp = {0}; - SRpcMsg reqMsg = *pMsg; - SRpcMsg rspMsg = {0}; - void *pRsp = NULL; + SBatchRsp batchRsp = {0}; + SRpcMsg reqMsg = *pMsg; + SRpcMsg rspMsg = {0}; + void *pRsp = NULL; if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -291,7 +291,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - int32_t msgNum = taosArrayGetSize(batchReq.pMsgs); + int32_t msgNum = taosArrayGetSize(batchReq.pMsgs); if (msgNum >= MAX_META_MSG_IN_BATCH) { code = TSDB_CODE_INVALID_MSG; qError("too many msgs %d in vnode batch meta req", msgNum); @@ -405,7 +405,8 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64, "nInsert"); VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64, "nInsertSuccess"); VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64, "nBatchInsert"); - VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64, "nBatchInsertSuccess"); + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64, + "nBatchInsertSuccess"); } void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 448726161710cddabed9231554af47f4332b22db..71458acce2cdab30365c452d23efd5841bb9424a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -349,7 +349,7 @@ int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot * snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path); } - vnodeCommitInfo(dir, &pWriter->info); + vnodeCommitInfo(dir); } else { vnodeRollback(pWriter->pVnode); } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 500e174421eacba6fc3c9ea203b945afb809f571..8651478afa58f4bf1805572d6e28fc3d09095185 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,7 +13,11 @@ * along with this program. If not, see . */ +#include "tencode.h" +#include "tmsg.h" #include "vnd.h" +#include "vnode.h" +#include "vnodeInt.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -24,145 +28,271 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); -int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { - int32_t code = 0; +static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { + int32_t code = 0; + int32_t lino = 0; + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // flags + if (tDecodeI32v(pCoder, NULL) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // name + char *name = NULL; + if (tDecodeCStr(pCoder, &name) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // uid + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, name); + if (uid == 0) { + uid = tGenIdPI64(); + } + *(int64_t *)(pCoder->data + pCoder->pos) = uid; + + // ctime + *(int64_t *)(pCoder->data + pCoder->pos + 8) = ctime; + + tEndDecode(pCoder); + +_exit: + if (code) { + vError("vgId:%d %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } else { + vTrace("vgId:%d %s done, table:%s uid generated:%" PRId64, TD_VID(pVnode), __func__, name, uid); + if (pUid) *pUid = uid; + } + return code; +} +static int32_t vnodePreProcessCreateTableMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + int64_t ctime = taosGetTimestampMs(); SDecoder dc = {0}; + int32_t nReqs; - switch (pMsg->msgType) { - case TDMT_VND_CREATE_TABLE: { - int64_t ctime = taosGetTimestampMs(); - int32_t nReqs; + tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); + if (tStartDecode(&dc) < 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } - tDecoderInit(&dc, (uint8_t *)pMsg->pCont + sizeof(SMsgHead), pMsg->contLen - sizeof(SMsgHead)); - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } + if (tDecodeI32v(&dc, &nReqs) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + for (int32_t iReq = 0; iReq < nReqs; iReq++) { + code = vnodePreprocessCreateTableReq(pVnode, &dc, ctime, NULL); + TSDB_CHECK_CODE(code, lino, _exit); + } - if (tDecodeI32v(&dc, &nReqs) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } - for (int32_t iReq = 0; iReq < nReqs; iReq++) { - tb_uid_t uid = tGenIdPI64(); - char *name = NULL; - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - goto _err; - } + tEndDecode(&dc); - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; +_exit: + tDecoderClear(&dc); + return code; +} +extern int64_t tsMaxKeyByPrecision[]; +static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int64_t ctime) { + int32_t code = 0; + int32_t lino = 0; - vTrace("vgId:%d, table:%s uid:%" PRId64 " is generated", pVnode->config.vgId, name, uid); - tEndDecode(&dc); - } + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } - tEndDecode(&dc); - tDecoderClear(&dc); - } break; - case TDMT_VND_SUBMIT: { - SSubmitMsgIter msgIter = {0}; - SSubmitReq *pSubmitReq = (SSubmitReq *)pMsg->pCont; - SSubmitBlk *pBlock = NULL; - int64_t ctime = taosGetTimestampMs(); - tb_uid_t uid; - - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - code = terrno; - goto _err; - } + SSubmitTbData submitTbData; + if (tDecodeI32v(pCoder, &submitTbData.flags) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; - - if (msgIter.schemaLen > 0) { - char *name = NULL; - - tDecoderInit(&dc, pBlock->data, msgIter.schemaLen); - if (tStartDecode(&dc) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - - if (tDecodeI32v(&dc, NULL) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - if (tDecodeCStr(&dc, &name) < 0) { - code = TSDB_CODE_INVALID_MSG; - return code; - } - - uid = metaGetTableEntryUidByName(pVnode->pMeta, name); - if (uid == 0) { - uid = tGenIdPI64(); - } - *(int64_t *)(dc.data + dc.pos) = uid; - *(int64_t *)(dc.data + dc.pos + 8) = ctime; - pBlock->uid = htobe64(uid); - - tEndDecode(&dc); - tDecoderClear(&dc); - } + int64_t uid; + if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + code = vnodePreprocessCreateTableReq(pVnode, pCoder, ctime, &uid); + TSDB_CHECK_CODE(code, lino, _exit); + } + + // submit data + if (tDecodeI64(pCoder, &submitTbData.suid) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { + *(int64_t *)(pCoder->data + pCoder->pos) = uid; + pCoder->pos += sizeof(int64_t); + } else { + if (tDecodeI64(pCoder, &submitTbData.uid) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + } + + if (tDecodeI32v(pCoder, &submitTbData.sver) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // scan and check + TSKEY now = ctime; + if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_MICRO) { + now *= 1000; + } else if (pVnode->config.tsdbCfg.precision == TSDB_TIME_PRECISION_NANO) { + now *= 1000000; + } + TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * pVnode->config.tsdbCfg.keep2; + TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision]; + if (submitTbData.flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + uint64_t nColData; + if (tDecodeU64v(pCoder, &nColData) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + SColData colData = {0}; + pCoder->pos += tGetColData(pCoder->data + pCoder->pos, &colData); + + if (colData.flag != HAS_VALUE) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t iRow = 0; iRow < colData.nVal; iRow++) { + if (((TSKEY *)colData.pData)[iRow] < minKey || ((TSKEY *)colData.pData)[iRow] > maxKey) { + code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + goto _exit; } + } + } else { + uint64_t nRow; + if (tDecodeU64v(pCoder, &nRow) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } - } break; - case TDMT_VND_DELETE: { - int32_t size; - int32_t ret; - uint8_t *pCont; - SEncoder *pCoder = &(SEncoder){0}; - SDeleteRes res = {0}; - SReadHandle handle = { - .meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; - - code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); - if (code) { - goto _err; + for (int32_t iRow = 0; iRow < nRow; ++iRow) { + SRow *pRow = (SRow *)(pCoder->data + pCoder->pos); + pCoder->pos += pRow->len; + + if (pRow->ts < minKey || pRow->ts > maxKey) { + code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + goto _exit; } + } + } + + tEndDecode(pCoder); + +_exit: + return code; +} +static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + int32_t lino = 0; + + SDecoder *pCoder = &(SDecoder){0}; + + tDecoderInit(pCoder, (uint8_t *)pMsg->pCont + sizeof(SSubmitReq2Msg), pMsg->contLen - sizeof(SSubmitReq2Msg)); + + if (tStartDecode(pCoder) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + uint64_t nSubmitTbData; + if (tDecodeU64v(pCoder, &nSubmitTbData) < 0) { + code = TSDB_CODE_INVALID_MSG; + TSDB_CHECK_CODE(code, lino, _exit); + } + + int64_t ctime = taosGetTimestampMs(); + for (int32_t i = 0; i < nSubmitTbData; i++) { + code = vnodePreProcessSubmitTbData(pVnode, pCoder, ctime); + TSDB_CHECK_CODE(code, lino, _exit); + } + + tEndDecode(pCoder); - // malloc and encode - tEncodeSize(tEncodeDeleteRes, &res, size, ret); - pCont = rpcMallocCont(size + sizeof(SMsgHead)); +_exit: + tDecoderClear(pCoder); + return code; +} + +static int32_t vnodePreProcessDeleteMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + + int32_t size; + int32_t ret; + uint8_t *pCont; + SEncoder *pCoder = &(SEncoder){0}; + SDeleteRes res = {0}; + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; + + code = qWorkerProcessDeleteMsg(&handle, pVnode->pQuery, pMsg, &res); + if (code) goto _exit; + + // malloc and encode + tEncodeSize(tEncodeDeleteRes, &res, size, ret); + pCont = rpcMallocCont(size + sizeof(SMsgHead)); - ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); - ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); + ((SMsgHead *)pCont)->contLen = size + sizeof(SMsgHead); + ((SMsgHead *)pCont)->vgId = TD_VID(pVnode); - tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); - tEncodeDeleteRes(pCoder, &res); - tEncoderClear(pCoder); + tEncoderInit(pCoder, pCont + sizeof(SMsgHead), size); + tEncodeDeleteRes(pCoder, &res); + tEncoderClear(pCoder); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = pCont; - pMsg->contLen = size + sizeof(SMsgHead); + rpcFreeCont(pMsg->pCont); + pMsg->pCont = pCont; + pMsg->contLen = size + sizeof(SMsgHead); - taosArrayDestroy(res.uidList); + taosArrayDestroy(res.uidList); + +_exit: + return code; +} + +int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) { + int32_t code = 0; + + switch (pMsg->msgType) { + case TDMT_VND_CREATE_TABLE: { + code = vnodePreProcessCreateTableMsg(pVnode, pMsg); + } break; + case TDMT_VND_SUBMIT: { + code = vnodePreProcessSubmitMsg(pVnode, pMsg); + } break; + case TDMT_VND_DELETE: { + code = vnodePreProcessDeleteMsg(pVnode, pMsg); } break; default: break; } - return code; - -_err: - vError("vgId%d, preprocess request failed since %s", TD_VID(pVnode), tstrerror(code)); +_exit: + if (code) { + vError("vgId%d failed to preprocess write request since %s, msg type:%d", TD_VID(pVnode), tstrerror(code), + pMsg->msgType); + } return code; } @@ -171,13 +301,13 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp void *pReq; int32_t len; int32_t ret; - + /* if (!pVnode->inUse) { terrno = TSDB_CODE_VND_NO_AVAIL_BUFPOOL; vError("vgId:%d, not ready to write since %s", TD_VID(pVnode), terrstr()); return -1; } - + */ if (version <= pVnode->state.applied) { vError("vgId:%d, duplicate write request. version: %" PRId64 ", applied: %" PRId64 "", TD_VID(pVnode), version, pVnode->state.applied); @@ -293,15 +423,21 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; - case TDMT_VND_ALTER_HASHRANGE: - vnodeProcessAlterHashRangeReq(pVnode, version, pReq, len, pRsp); - break; case TDMT_VND_ALTER_CONFIG: vnodeProcessAlterConfigReq(pVnode, version, pReq, len, pRsp); break; case TDMT_VND_COMMIT: needCommit = true; break; + case TDMT_VND_CREATE_INDEX: + vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp); + break; + case TDMT_VND_DROP_INDEX: + vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp); + break; + case TDMT_VND_COMPACT: + vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp); + goto _exit; default: vError("vgId:%d, unprocessed msg, %d", TD_VID(pVnode), pMsg->msgType); return -1; @@ -323,7 +459,10 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp // commit if need if (needCommit) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); - vnodeAsyncCommit(pVnode); + if (vnodeAsyncCommit(pVnode) < 0) { + vError("vgId:%d, failed to vnode async commit since %s.", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } // start a new one if (vnodeBegin(pVnode) < 0) { @@ -489,6 +628,15 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p tqUpdateTbUidList(pVnode->pTq, tbUids, false); } +#if 0 + // process + ret = tsdbDoRetention(pVnode->pTsdb, ttlReq.timestamp); + if (ret) goto end; + + ret = smaDoRetention(pVnode->pSma, ttlReq.timestamp); + if (ret) goto end; +#endif + end: taosArrayDestroy(tbUids); return ret; @@ -852,176 +1000,371 @@ static int32_t vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, return TSDB_CODE_SUCCESS; } -static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char *tags) { - ASSERT(pMsg != NULL); - SSubmitMsgIter msgIter = {0}; - SMeta *pMeta = pVnode->pMeta; - SSubmitBlk *pBlock = NULL; +typedef struct SSubmitReqConvertCxt { + SSubmitMsgIter msgIter; + SSubmitBlk *pBlock; + SSubmitBlkIter blkIter; + STSRow *pRow; + STSRowIter rowIter; + SSubmitTbData *pTbData; + STSchema *pTbSchema; + SArray *pColValues; +} SSubmitReqConvertCxt; + +static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) { + taosMemoryFreeClear(pCxt->pTbSchema); + pCxt->pTbSchema = metaGetTbTSchema(pMeta, pCxt->msgIter.suid > 0 ? pCxt->msgIter.suid : pCxt->msgIter.uid, + pCxt->msgIter.sversion, 1); + if (NULL == pCxt->pTbSchema) { + return TSDB_CODE_INVALID_MSG; + } + tdSTSRowIterInit(&pCxt->rowIter, pCxt->pTbSchema); - if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; - while (true) { - if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; - if (pBlock == NULL) break; + tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); + if (NULL == pCxt->pTbData) { + pCxt->pTbData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pCxt->pTbData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + pCxt->pTbData->flags = 0; + pCxt->pTbData->suid = pCxt->msgIter.suid; + pCxt->pTbData->uid = pCxt->msgIter.uid; + pCxt->pTbData->sver = pCxt->msgIter.sversion; + pCxt->pTbData->pCreateTbReq = NULL; + pCxt->pTbData->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pCxt->pTbData->aRowP) { + return TSDB_CODE_OUT_OF_MEMORY; + } - vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags); + taosArrayDestroy(pCxt->pColValues); + pCxt->pColValues = taosArrayInit(pCxt->pTbSchema->numOfCols, sizeof(SColVal)); + if (NULL == pCxt->pColValues) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < pCxt->pTbSchema->numOfCols; ++i) { + SColVal val = COL_VAL_NONE(pCxt->pTbSchema->columns[i].colId, pCxt->pTbSchema->columns[i].type); + taosArrayPush(pCxt->pColValues, &val); } - return 0; + return TSDB_CODE_SUCCESS; } -static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp submitRsp = {0}; - SSubmitMsgIter msgIter = {0}; - SSubmitBlk *pBlock = NULL; - SVCreateTbReq createTbReq = {0}; - SDecoder decoder = {0}; - int32_t nRows = 0; - int32_t tsize, ret; - SEncoder encoder = {0}; - SArray *newTbUids = NULL; - SVStatis statis = {0}; - bool tbCreated = false; - terrno = TSDB_CODE_SUCCESS; - - pRsp->code = 0; - pSubmitReq->version = version; - statis.nBatchInsert = 1; - -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSubmitMsg(pVnode, pReq, __func__); -#endif +static void vnodeDestroySubmitReqConvertCxt(SSubmitReqConvertCxt *pCxt) { + taosMemoryFreeClear(pCxt->pTbSchema); + tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); + taosMemoryFreeClear(pCxt->pTbData); + taosArrayDestroy(pCxt->pColValues); +} - if (tsdbScanAndConvertSubmitMsg(pVnode->pTsdb, pSubmitReq) < 0) { - pRsp->code = terrno; - goto _exit; +static int32_t vnodeCellValConvertToColVal(STColumn *pCol, SCellVal *pCellVal, SColVal *pColVal) { + if (tdValTypeIsNone(pCellVal->valType)) { + pColVal->flag = CV_FLAG_NONE; + return TSDB_CODE_SUCCESS; } - // handle the request - if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - goto _exit; + if (tdValTypeIsNull(pCellVal->valType)) { + pColVal->flag = CV_FLAG_NULL; + return TSDB_CODE_SUCCESS; } - submitRsp.pArray = taosArrayInit(msgIter.numOfBlocks, sizeof(SSubmitBlkRsp)); - newTbUids = taosArrayInit(msgIter.numOfBlocks, sizeof(int64_t)); - if (!submitRsp.pArray || !newTbUids) { - pRsp->code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + if (IS_VAR_DATA_TYPE(pCol->type)) { + pColVal->value.nData = varDataLen(pCellVal->val); + pColVal->value.pData = varDataVal(pCellVal->val); + } else if (TSDB_DATA_TYPE_FLOAT == pCol->type) { + float f = GET_FLOAT_VAL(pCellVal->val); + memcpy(&pColVal->value.val, &f, sizeof(f)); + } else if (TSDB_DATA_TYPE_DOUBLE == pCol->type) { + pColVal->value.val = *(int64_t *)pCellVal->val; + } else { + GET_TYPED_DATA(pColVal->value.val, int64_t, pCol->type, pCellVal->val); + } + + pColVal->flag = CV_FLAG_VALUE; + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeTSRowConvertToColValArray(SSubmitReqConvertCxt *pCxt) { + int32_t code = TSDB_CODE_SUCCESS; + tdSTSRowIterReset(&pCxt->rowIter, pCxt->pRow); + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < pCxt->pTbSchema->numOfCols; ++i) { + STColumn *pCol = pCxt->pTbSchema->columns + i; + SCellVal cellVal = {0}; + if (!tdSTSRowIterFetch(&pCxt->rowIter, pCol->colId, pCol->type, &cellVal)) { + break; + } + code = vnodeCellValConvertToColVal(pCol, &cellVal, (SColVal *)taosArrayGet(pCxt->pColValues, i)); + } + return code; +} + +static int32_t vnodeDecodeCreateTbReq(SSubmitReqConvertCxt *pCxt) { + if (pCxt->msgIter.schemaLen <= 0) { + return TSDB_CODE_SUCCESS; + } + + pCxt->pTbData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCxt->pTbData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDecoder decoder = {0}; + tDecoderInit(&decoder, pCxt->pBlock->data, pCxt->msgIter.schemaLen); + int32_t code = tDecodeSVCreateTbReq(&decoder, pCxt->pTbData->pCreateTbReq); + tDecoderClear(&decoder); + + return code; +} + +static int32_t vnodeSubmitReqConvertToSubmitReq2(SVnode *pVnode, SSubmitReq *pReq, SSubmitReq2 *pReq2) { + pReq2->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); + if (NULL == pReq2->aSubmitTbData) { + return TSDB_CODE_OUT_OF_MEMORY; } - for (;;) { - tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; + SSubmitReqConvertCxt cxt = {0}; - SSubmitBlkRsp submitBlkRsp = {0}; - tbCreated = false; + int32_t code = tInitSubmitMsgIter(pReq, &cxt.msgIter); + while (TSDB_CODE_SUCCESS == code) { + code = tGetSubmitMsgNext(&cxt.msgIter, &cxt.pBlock); + if (TSDB_CODE_SUCCESS == code) { + if (NULL == cxt.pBlock) { + break; + } + code = vnodeResetTableCxt(pVnode->pMeta, &cxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = tInitSubmitBlkIter(&cxt.msgIter, cxt.pBlock, &cxt.blkIter); + } + if (TSDB_CODE_SUCCESS == code) { + code = vnodeDecodeCreateTbReq(&cxt); + } + while (TSDB_CODE_SUCCESS == code && (cxt.pRow = tGetSubmitBlkNext(&cxt.blkIter)) != NULL) { + code = vnodeTSRowConvertToColValArray(&cxt); + if (TSDB_CODE_SUCCESS == code) { + SRow **pNewRow = taosArrayReserve(cxt.pTbData->aRowP, 1); + code = tRowBuild(cxt.pColValues, cxt.pTbSchema, pNewRow); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = (NULL == taosArrayPush(pReq2->aSubmitTbData, cxt.pTbData) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + } + if (TSDB_CODE_SUCCESS == code) { + taosMemoryFreeClear(cxt.pTbData); + } + } + + vnodeDestroySubmitReqConvertCxt(&cxt); + return code; +} + +static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { + int32_t code = TSDB_CODE_SUCCESS; + char *pMsg = NULL; + uint32_t msglen = 0; + tEncodeSize(tEncodeSSubmitReq2, pSubmitReq, msglen, code); + if (TSDB_CODE_SUCCESS == code) { + pMsg = taosMemoryMalloc(msglen); + if (NULL == pMsg) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + tEncoderInit(&encoder, pMsg, msglen); + code = tEncodeSSubmitReq2(&encoder, pSubmitReq); + tEncoderClear(&encoder); + } + if (TSDB_CODE_SUCCESS == code) { + *ppMsg = pMsg; + } + return code; +} + +static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + int32_t code = 0; + terrno = 0; - // create table for auto create table mode - if (msgIter.schemaLen > 0) { - tDecoderInit(&decoder, pBlock->data, msgIter.schemaLen); - if (tDecodeSVCreateTbReq(&decoder, &createTbReq) < 0) { - pRsp->code = TSDB_CODE_INVALID_MSG; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + SSubmitReq2 *pSubmitReq = &(SSubmitReq2){0}; + SSubmitRsp2 *pSubmitRsp = &(SSubmitRsp2){0}; + SArray *newTbUids = NULL; + int32_t ret; + SEncoder ec = {0}; + + pRsp->code = TSDB_CODE_SUCCESS; + + void *pAllocMsg = NULL; + SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq; + if (0 == pMsg->version) { + code = vnodeSubmitReqConvertToSubmitReq2(pVnode, (SSubmitReq *)pMsg, pSubmitReq); + if (TSDB_CODE_SUCCESS == code) { + code = vnodeRebuildSubmitReqMsg(pSubmitReq, &pReq); + } + if (TSDB_CODE_SUCCESS == code) { + pAllocMsg = pReq; + } + if (TSDB_CODE_SUCCESS != code) { + goto _exit; + } + } else { + // decode + pReq = POINTER_SHIFT(pReq, sizeof(SSubmitReq2Msg)); + len -= sizeof(SSubmitReq2Msg); + SDecoder dc = {0}; + tDecoderInit(&dc, pReq, len); + if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + tDecoderClear(&dc); + } + + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); + + if (pSubmitTbData->pCreateTbReq) { + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; + } else { + SMetaInfo info = {0}; + + code = metaGetInfo(pVnode->pMeta, pSubmitTbData->uid, &info, NULL); + if (code) { + code = TSDB_CODE_TDB_TABLE_NOT_EXIST; + vWarn("vgId:%d, table uid:%" PRId64 " not exists", TD_VID(pVnode), pSubmitTbData->uid); + goto _exit; + } + + if (info.suid != pSubmitTbData->suid) { + code = TSDB_CODE_INVALID_MSG; goto _exit; } - if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + if (info.suid) { + metaGetInfo(pVnode->pMeta, info.suid, &info, NULL); + } + + if (pSubmitTbData->sver != info.skmVer) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; goto _exit; } + } + + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + int32_t nColData = TARRAY_SIZE(pSubmitTbData->aCol); + SColData *aColData = (SColData *)TARRAY_DATA(pSubmitTbData->aCol); - if ((terrno = grantCheck(TSDB_GRANT_TABLE)) < 0) { - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + if (nColData <= 0) { + code = TSDB_CODE_INVALID_MSG; goto _exit; } - if (metaCreateTable(pVnode->pMeta, version, &createTbReq, &submitBlkRsp.pMeta) < 0) { - if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { - submitBlkRsp.code = terrno; - pRsp->code = terrno; - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); + if (aColData[0].cid != PRIMARYKEY_TIMESTAMP_COL_ID || aColData[0].type != TSDB_DATA_TYPE_TIMESTAMP || + aColData[0].nVal <= 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + for (int32_t i = 1; i < nColData; i++) { + if (aColData[i].nVal != aColData[0].nVal) { + code = TSDB_CODE_INVALID_MSG; goto _exit; } - } else { - if (NULL != submitBlkRsp.pMeta) { - vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); - } + } + } + } - taosArrayPush(newTbUids, &createTbReq.uid); + vDebug("vgId:%d, submit block size %d", TD_VID(pVnode), (int32_t)taosArrayGetSize(pSubmitReq->aSubmitTbData)); - submitBlkRsp.uid = createTbReq.uid; - submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); - sprintf(submitBlkRsp.tblFName, "%s.%s", pVnode->config.dbname, createTbReq.name); - tbCreated = true; - } + // loop to handle + for (int32_t i = 0; i < TARRAY_SIZE(pSubmitReq->aSubmitTbData); ++i) { + SSubmitTbData *pSubmitTbData = taosArrayGet(pSubmitReq->aSubmitTbData, i); - msgIter.uid = createTbReq.uid; - if (createTbReq.type == TSDB_CHILD_TABLE) { - msgIter.suid = createTbReq.ctb.suid; - } else { - msgIter.suid = 0; + // create table + if (pSubmitTbData->pCreateTbReq) { + // check (TODO: move check to create table) + code = grantCheck(TSDB_GRANT_TIMESERIES); + if (code) goto _exit; + + code = grantCheck(TSDB_GRANT_TABLE); + if (code) goto _exit; + + // alloc if need + if (pSubmitRsp->aCreateTbRsp == NULL && + (pSubmitRsp->aCreateTbRsp = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(SVCreateTbRsp))) == + NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } -#ifdef TD_DEBUG_PRINT_ROW - vnodeDebugPrintSingleSubmitMsg(pVnode->pMeta, pBlock, &msgIter, "real uid"); -#endif - tDecoderClear(&decoder); - taosArrayDestroy(createTbReq.ctb.tagName); - } + SVCreateTbRsp *pCreateTbRsp = taosArrayReserve(pSubmitRsp->aCreateTbRsp, 1); - if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) { - submitBlkRsp.code = terrno; - } + // create table + if (metaCreateTable(pVnode->pMeta, version, pSubmitTbData->pCreateTbReq, &pCreateTbRsp->pMeta) == + 0) { // create table success + + if (newTbUids == NULL && + (newTbUids = taosArrayInit(TARRAY_SIZE(pSubmitReq->aSubmitTbData), sizeof(int64_t))) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + taosArrayPush(newTbUids, &pSubmitTbData->uid); - submitRsp.numOfRows += submitBlkRsp.numOfRows; - submitRsp.affectedRows += submitBlkRsp.affectedRows; - if (tbCreated || submitBlkRsp.code) { - taosArrayPush(submitRsp.pArray, &submitBlkRsp); + if (pCreateTbRsp->pMeta) { + vnodeUpdateMetaRsp(pVnode, pCreateTbRsp->pMeta); + } + } else { // create table failed + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + code = terrno; + goto _exit; + } + pSubmitTbData->uid = pSubmitTbData->pCreateTbReq->uid; // update uid if table exist for using below + } } + + // insert data + int32_t affectedRows; + code = tsdbInsertTableData(pVnode->pTsdb, version, pSubmitTbData, &affectedRows); + if (code) goto _exit; + + pSubmitRsp->affectedRows += affectedRows; } + // update table uid list if (taosArrayGetSize(newTbUids) > 0) { vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); + tqUpdateTbUidList(pVnode->pTq, newTbUids, true); } - tqUpdateTbUidList(pVnode->pTq, newTbUids, true); - _exit: - taosArrayDestroy(newTbUids); - tEncodeSize(tEncodeSSubmitRsp, &submitRsp, tsize, ret); - pRsp->pCont = rpcMallocCont(tsize); - pRsp->contLen = tsize; - tEncoderInit(&encoder, pRsp->pCont, tsize); - tEncodeSSubmitRsp(&encoder, &submitRsp); - tEncoderClear(&encoder); - - taosArrayDestroyEx(submitRsp.pArray, tFreeSSubmitBlkRsp); + // message + pRsp->code = code; + tEncodeSize(tEncodeSSubmitRsp2, pSubmitRsp, pRsp->contLen, ret); + pRsp->pCont = rpcMallocCont(pRsp->contLen); + tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); + tEncodeSSubmitRsp2(&ec, pSubmitRsp); + tEncoderClear(&ec); - // TODO: the partial success scenario and the error case - // => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level - // 1/level 2. - // TODO: refactor - if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { - statis.nBatchInsertSuccess = 1; - tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); + // update statistics + atomic_add_fetch_64(&pVnode->statis.nInsert, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); + atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); + if (code == 0) { + atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); + tdProcessRSmaSubmit(pVnode->pSma, version, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } - // N.B. not strict as the following procedure is not atomic - atomic_add_fetch_64(&pVnode->statis.nInsert, submitRsp.numOfRows); - atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, submitRsp.affectedRows); - atomic_add_fetch_64(&pVnode->statis.nBatchInsert, statis.nBatchInsert); - atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, statis.nBatchInsertSuccess); + // clear + taosArrayDestroy(newTbUids); + tDestroySSubmitReq2(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); + tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); - vDebug("vgId:%d %s done, index:%" PRId64, TD_VID(pVnode), __func__, version); - return 0; + if (code) terrno = code; + + taosMemoryFree(pAllocMsg); + + return code; } static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { @@ -1083,16 +1426,6 @@ static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void return 0; } -static int32_t vnodeProcessAlterHashRangeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - vInfo("vgId:%d, alter hashrange msg will be processed", TD_VID(pVnode)); - - // todo - // 1. stop work - // 2. adjust hash range / compact / remove wals / rename vgroups - // 3. reload sync - return 0; -} - static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { bool walChanged = false; bool tsdbChanged = false; @@ -1255,3 +1588,61 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq _err: return code; } +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVCreateStbReq req = {0}; + SDecoder dc = {0}; + + pRsp->msgType = TDMT_VND_CREATE_INDEX_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + tDecoderInit(&dc, pReq, len); + // decode req + if (tDecodeSVCreateStbReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } + if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + tDecoderClear(&dc); + return 0; +_err: + tDecoderClear(&dc); + return -1; +} +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SDropIndexReq req = {0}; + pRsp->msgType = TDMT_VND_CREATE_INDEX_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + if (tDeserializeSDropIdxReq(pReq, len, &req)) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + return -1; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SCompactVnodeReq req = {0}; + if (tDeserializeSCompactVnodeReq(pReq, len, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return TSDB_CODE_INVALID_MSG; + } + vInfo("vgId:%d, compact msg will be processed, db:%s dbUid:%" PRId64 " compactStartTime:%" PRId64, TD_VID(pVnode), + req.db, req.dbUid, req.compactStartTime); + + vnodeAsyncCompact(pVnode); + vnodeBegin(pVnode); + + return 0; +} diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 6f3788616a5f45e440d07f23b89e3e493d07fa4d..1e96a76170ca3eb972a9199d41c47e512c11639d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -219,7 +219,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) isWeak, isBlock, msg, numOfMsgs, arrayPos, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); + vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_SYN_RESTORING; vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); @@ -228,7 +228,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) } if (pMsgArr == NULL || pIsWeakArr == NULL) { - vGError("vgId:%d, msg:%p failed to process since out of memory", vgId, pMsg); + vGError("vgId:%d, msg:%p failed to process since out of memory, type:%s", vgId, pMsg, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_OUT_OF_MEMORY; vnodeHandleProposeError(pVnode, pMsg, terrno); rpcFreeCont(pMsg->pCont); @@ -281,7 +281,7 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) vnodeIsMsgBlock(pMsg->msgType), msg, numOfMsgs, pMsg->info.handle); if (!pVnode->restored) { - vGError("vgId:%d, msg:%p failed to process since restore not finished", vgId, pMsg); + vGError("vgId:%d, msg:%p failed to process since restore not finished, type:%s", vgId, pMsg, TMSG_INFO(pMsg->msgType)); vnodeHandleProposeError(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); @@ -635,7 +635,7 @@ int32_t vnodeSyncStart(SVnode *pVnode) { } void vnodeSyncPreClose(SVnode *pVnode) { - vInfo("vgId:%d, pre close sync", pVnode->config.vgId); + vInfo("vgId:%d, sync pre close", pVnode->config.vgId); syncLeaderTransfer(pVnode->sync); syncPreStop(pVnode->sync); @@ -649,7 +649,7 @@ void vnodeSyncPreClose(SVnode *pVnode) { } void vnodeSyncPostClose(SVnode *pVnode) { - vInfo("vgId:%d, post close sync", pVnode->config.vgId); + vInfo("vgId:%d, sync post close", pVnode->config.vgId); syncPostStop(pVnode->sync); } diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 72459f4d35437c704f7021d14126f6166388bbe5..e18be94ace94c35a486a7ebd8c54e4eb3ae2e009 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,5 +1,4 @@ add_subdirectory(tdb) -add_subdirectory(cache) add_subdirectory(transport) add_subdirectory(wal) add_subdirectory(monitor) diff --git a/source/libs/cache/CMakeLists.txt b/source/libs/cache/CMakeLists.txt deleted file mode 100644 index dc631e5bfd4126fe2edbc057668430043a43f31c..0000000000000000000000000000000000000000 --- a/source/libs/cache/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -aux_source_directory(src CACHE_SRC) -add_library(cache STATIC ${CACHE_SRC}) -target_include_directories( - cache - PUBLIC "${TD_SOURCE_DIR}/include/libs/cache" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file diff --git a/source/libs/cache/src/cache.c b/source/libs/cache/src/cache.c deleted file mode 100644 index 6dea4a4e57392be988126c579648f39a8270b9bf..0000000000000000000000000000000000000000 --- a/source/libs/cache/src/cache.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 41c7eca7f09ad01c59f694c64f6b554f0a705157..8fc7df63be2204f872bcf0eca8def42deb819d9e 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -582,34 +582,34 @@ typedef struct SCtgOperation { #define CTG_LOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define CTG_UNLOCK(type, _lock) \ do { \ if (CTG_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 06db2c3268a298c71d33586eead38643cae40d58..7ff8afd6a510f16c6a47df25550c9d2dbd0e3543 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -493,11 +493,9 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - if (tbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - taosHashRelease(dbCache->tbCache, tbCache); - *pTb = NULL; - } + CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosHashRelease(dbCache->tbCache, tbCache); + *pTb = NULL; ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); @@ -1554,8 +1552,8 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam SCtgTbCache cache = {0}; cache.pMeta = meta; if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) { - taosMemoryFree(meta); ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + taosMemoryFree(meta); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index b6ff1c8b987278ee8ef223577e91771c12cefde8..6b870232c74bdb09dfd90bbb95a9cae8954f23da 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -22,7 +22,6 @@ extern SCatalogMgmt gCtgMgmt; SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData *pResult, void *param, int32_t code) { - ASSERT(*(int32_t *)param == 1); taosMemoryFree(param); qDebug("async call result: %s", tstrerror(code)); @@ -282,7 +281,7 @@ int32_t ctgdHandleDbgCommand(char *command) { CTG_RET(TSDB_CODE_INVALID_PARA); } - char *dup = strdup(command); + char *dup = taosStrdup(command); char *option = NULL; char *param = NULL; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 7cc6c90d303c4c425358800eb646f986417cefdd..3b037e206290ff767a1f879e0b46aadf8b3e4f20 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -40,7 +40,9 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu msgNum = taosArrayGetSize(batchRsp.pRsps); } - ASSERT(taskNum == msgNum || 0 == msgNum); + if (ASSERTS(taskNum == msgNum || 0 == msgNum, "taskNum %d mis-match msgNum %d", taskNum, msgNum)) { + msgNum = 0; + } ctgDebug("QID:0x%" PRIx64 " ctg got batch %d rsp %s", pJob->queryId, cbParam->batchId, TMSG_INFO(cbParam->reqType + 1)); @@ -58,11 +60,19 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); - taskMsg.msgType = pRsp->reqType; - taskMsg.pData = pRsp->msg; - taskMsg.len = pRsp->msgLen; - - ASSERT(pRsp->msgIdx == *msgIdx); + if (ASSERTS(pRsp->msgIdx == *msgIdx, "rsp msgIdx %d mis-match msgIdx %d", pRsp->msgIdx, *msgIdx)) { + pRsp = &rsp; + pRsp->msgIdx = *msgIdx; + pRsp->reqType = -1; + pRsp->rspCode = 0; + taskMsg.msgType = -1; + taskMsg.pData = NULL; + taskMsg.len = 0; + } else { + taskMsg.msgType = pRsp->reqType; + taskMsg.pData = pRsp->msg; + taskMsg.len = pRsp->msgLen; + } } else { pRsp = &rsp; pRsp->msgIdx = *msgIdx; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 3dd40a413996c458dc204b851ae08e6ca4280529..cd9380778b598080405ca569672e0c7b27d5fc72 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -742,7 +742,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ pCtx->reqType = reqType; pCtx->out = out; if (target) { - pCtx->target = strdup(target); + pCtx->target = taosStrdup(target); if (NULL == pCtx->target) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -759,7 +759,7 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { ctx.reqType = reqType; ctx.out = out; if (target) { - ctx.target = strdup(target); + ctx.target = taosStrdup(target); if (NULL == ctx.target) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -1169,7 +1169,7 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { for (int32_t i = 0; i < num; ++i) { STableIndexInfo* pInfo = taosArrayGet(pIndex, i); pInfo = taosArrayPush(*pRes, pInfo); - pInfo->expr = strdup(pInfo->expr); + pInfo->expr = taosStrdup(pInfo->expr); } return TSDB_CODE_SUCCESS; @@ -1179,7 +1179,7 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, cha if (msgType == TDMT_VND_TABLE_META || msgType == TDMT_VND_TABLE_CFG || msgType == TDMT_VND_BATCH_META) { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; - pMsgSendInfo->target.dbFName = strdup(dbFName); + pMsgSendInfo->target.dbFName = taosStrdup(dbFName); } else { pMsgSendInfo->target.type = TARGET_TYPE_MNODE; } diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index 6acf19218d893a9962853e1fe7b0c73c8dddf7bc..2a435b43e8bd645c18a3122d6ab042ac57ec9077 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -64,6 +64,9 @@ extern "C" { #define EXPLAIN_IGNORE_GROUPID_FORMAT "Ignore Group Id: %s" #define EXPLAIN_PARTITION_KETS_FORMAT "Partition Key: " #define EXPLAIN_INTERP_FORMAT "Interp" +#define EXPLAIN_EVENT_FORMAT "Event" +#define EXPLAIN_EVENT_START_FORMAT "Start Cond: " +#define EXPLAIN_EVENT_END_FORMAT "End Cond: " #define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms" #define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms" @@ -98,6 +101,8 @@ extern "C" { #define COMMAND_SCHEDULE_POLICY "schedulePolicy" #define COMMAND_ENABLE_RESCHEDULE "enableReSchedule" #define COMMAND_CATALOG_DEBUG "catalogDebug" +#define COMMAND_ENABLE_MEM_DEBUG "enableMemDebug" +#define COMMAND_DISABLE_MEM_DEBUG "disableMemDebug" typedef struct SExplainGroup { int32_t nodeNum; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 6674ab2ef2bfca2876a19cd2169a40af96ef8d3f..f88701afe29892dba5ee3b17c463d2b44de9d981 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -21,6 +21,7 @@ #include "tdatablock.h" #include "tglobal.h" #include "tgrant.h" +#include "taosdef.h" extern SConfig* tsCfg; @@ -40,7 +41,6 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfCols = htonl(numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); return TSDB_CODE_SUCCESS; } @@ -154,6 +154,23 @@ static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) { return code; } +static int32_t buildAliveResultDataBlock(SSDataBlock** pOutput) { + SSDataBlock* pBlock = createDataBlock(); + if (NULL == pBlock) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + int32_t code = blockDataAppendColInfo(pBlock, &infoData); + + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pBlock; + } else { + blockDataDestroy(pBlock); + } + return code; +} + int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { int64_t v = 0; switch (unit) { @@ -281,6 +298,108 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S colDataSetVal(pCol2, 0, buf2, false); } +#define CHECK_LEADER(n) (row[n] && (fields[n].type == TSDB_DATA_TYPE_VARCHAR && strncasecmp(row[n], "leader", varDataLen((char *)row[n] - VARSTR_HEADER_SIZE)) == 0)) +// on this row, if have leader return true else return false +bool existLeaderRole(TAOS_ROW row, TAOS_FIELD* fields, int nFields) { + // vgroup_id | db_name | tables | v1_dnode | v1_status | v2_dnode | v2_status | v3_dnode | v3_status | v4_dnode | + // v4_status | cacheload | tsma | + if (nFields != 13) { + return false; + } + + // check have leader on cloumn v*_status on 4 6 8 10 + if (CHECK_LEADER(4) || CHECK_LEADER(6) || CHECK_LEADER(8) || CHECK_LEADER(10)) { + return true; + } + + return false; +} + +// get db alive status, return 1 is alive else return 0 +int32_t getAliveStatusFromApi(int64_t* pConnId, char* dbName, int32_t* pStatus) { + char sql[128 + TSDB_DB_NAME_LEN] = "select * from information_schema.ins_vgroups"; + int32_t code; + + // filter with db name + if (dbName && dbName[0] != 0) { + char str[64 + TSDB_DB_NAME_LEN] = ""; + // test db name exist + sprintf(str, "show create database %s ;", dbName); + TAOS_RES* dbRes = taos_query(pConnId, str); + code = taos_errno(dbRes); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(dbRes); + return code; + } + taos_free_result(dbRes); + + sprintf(str, " where db_name='%s' ;", dbName); + strcat(sql, str); + } + + TAOS_RES* res = taos_query(pConnId, sql); + code = taos_errno(res); + if (code != TSDB_CODE_SUCCESS) { + taos_free_result(res); + return code; + } + + TAOS_ROW row = NULL; + TAOS_FIELD* fields = taos_fetch_fields(res); + int32_t nFields = taos_num_fields(res); + int32_t nAvailble = 0; + int32_t nUnAvailble = 0; + + while ((row = taos_fetch_row(res)) != NULL) { + if (existLeaderRole(row, fields, nFields)) { + nAvailble++; + } else { + nUnAvailble++; + } + } + taos_free_result(res); + + int32_t status = 0; + if (nAvailble + nUnAvailble == 0 || nUnAvailble == 0) { + status = SHOW_STATUS_AVAILABLE; + } else if (nAvailble > 0 && nUnAvailble > 0) { + status = SHOW_STATUS_HALF_AVAILABLE; + } else { + status = SHOW_STATUS_NOT_AVAILABLE; + } + + if (pStatus) { + *pStatus = status; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setAliveResultIntoDataBlock(int64_t* pConnId, SSDataBlock* pBlock, char* dbName) { + blockDataEnsureCapacity(pBlock, 1); + pBlock->info.rows = 1; + + SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); + int32_t status = 0; + int32_t code = getAliveStatusFromApi(pConnId, dbName, &status); + if (code == TSDB_CODE_SUCCESS) { + colDataAppend(pCol1, 0, (const char*)&status, false); + } + return code; +} + +static int32_t execShowAliveStatus(int64_t* pConnId, SShowAliveStmt* pStmt, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = NULL; + int32_t code = buildAliveResultDataBlock(&pBlock); + if (TSDB_CODE_SUCCESS == code) { + code = setAliveResultIntoDataBlock(pConnId, pBlock, pStmt->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildRetrieveTableRsp(pBlock, SHOW_ALIVE_RESULT_COLS, pRsp); + } + blockDataDestroy(pBlock); + return code; +} + static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = NULL; int32_t code = buildCreateDBResultDataBlock(&pBlock); @@ -568,6 +687,21 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { code = schedulerEnableReSchedule(atoi(value)); } else if (0 == strcasecmp(cmd, COMMAND_CATALOG_DEBUG)) { code = ctgdHandleDbgCommand(value); + } else if (0 == strcasecmp(cmd, COMMAND_ENABLE_MEM_DEBUG)) { + code = taosMemoryDbgInit(); + if (code) { + qError("failed to init memory dbg, error:%s", tstrerror(code)); + return code; + } + tsAsyncLog = false; + qInfo("memory dbg enabled"); + } else if (0 == strcasecmp(cmd, COMMAND_DISABLE_MEM_DEBUG)) { + code = taosMemoryDbgInitRestore(); + if (code) { + qError("failed to restore from memory dbg, error:%s", tstrerror(code)); + return code; + } + qInfo("memory dbg disabled"); } else { goto _return; } @@ -736,7 +870,7 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p return code; } -int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { +int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: return execDescribe(sysInfoUser, pStmt, pRsp); @@ -754,6 +888,9 @@ int32_t qExecCommand(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp) { return execShowLocalVariables(pRsp); case QUERY_NODE_SELECT_STMT: return execSelectWithoutFrom((SSelectStmt*)pStmt, pRsp); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return execShowAliveStatus(pConnId, (SShowAliveStmt*)pStmt, pRsp); default: break; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index d3cc6e0d28f22c11f978608b3331ac617ad07f8c..4302302d7ab6226f2eda506500c9ba0c3bb706fd 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -114,129 +114,7 @@ _return: int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNodeList **pChildren) { int32_t tlen = 0; - SNodeList *pPhysiChildren = NULL; - - switch (pNode->type) { - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: { - STagScanPhysiNode *pTagScanNode = (STagScanPhysiNode *)pNode; - pPhysiChildren = pTagScanNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: { - STableScanPhysiNode *pTblScanNode = (STableScanPhysiNode *)pNode; - pPhysiChildren = pTblScanNode->scan.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: { - SSystemTableScanPhysiNode *pSTblScanNode = (SSystemTableScanPhysiNode *)pNode; - pPhysiChildren = pSTblScanNode->scan.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { - SProjectPhysiNode *pPrjNode = (SProjectPhysiNode *)pNode; - pPhysiChildren = pPrjNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: { - SSortMergeJoinPhysiNode *pJoinNode = (SSortMergeJoinPhysiNode *)pNode; - pPhysiChildren = pJoinNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { - SAggPhysiNode *pAggNode = (SAggPhysiNode *)pNode; - pPhysiChildren = pAggNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: { - SExchangePhysiNode *pExchNode = (SExchangePhysiNode *)pNode; - pPhysiChildren = pExchNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_SORT: { - SSortPhysiNode *pSortNode = (SSortPhysiNode *)pNode; - pPhysiChildren = pSortNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: { - SIntervalPhysiNode *pIntNode = (SIntervalPhysiNode *)pNode; - pPhysiChildren = pIntNode->window.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: { - SSessionWinodwPhysiNode *pSessNode = (SSessionWinodwPhysiNode *)pNode; - pPhysiChildren = pSessNode->window.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: { - SStateWinodwPhysiNode *pStateNode = (SStateWinodwPhysiNode *)pNode; - pPhysiChildren = pStateNode->window.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { - SPartitionPhysiNode *partitionPhysiNode = (SPartitionPhysiNode *)pNode; - pPhysiChildren = partitionPhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE: { - SMergePhysiNode *mergePhysiNode = (SMergePhysiNode *)pNode; - pPhysiChildren = mergePhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: { - SIndefRowsFuncPhysiNode *indefPhysiNode = (SIndefRowsFuncPhysiNode *)pNode; - pPhysiChildren = indefPhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { - SMergeAlignedIntervalPhysiNode *intPhysiNode = (SMergeAlignedIntervalPhysiNode *)pNode; - pPhysiChildren = intPhysiNode->window.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_FILL: { - SFillPhysiNode *fillPhysiNode = (SFillPhysiNode *)pNode; - pPhysiChildren = fillPhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: { - STableMergeScanPhysiNode *mergePhysiNode = (STableMergeScanPhysiNode *)pNode; - pPhysiChildren = mergePhysiNode->scan.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: { - SBlockDistScanPhysiNode *distPhysiNode = (SBlockDistScanPhysiNode *)pNode; - pPhysiChildren = distPhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: { - SLastRowScanPhysiNode *lastRowPhysiNode = (SLastRowScanPhysiNode *)pNode; - pPhysiChildren = lastRowPhysiNode->scan.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: { - STableCountScanPhysiNode *tableCountPhysiNode = (STableCountScanPhysiNode *)pNode; - pPhysiChildren = tableCountPhysiNode->scan.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: { - SGroupSortPhysiNode *groupSortPhysiNode = (SGroupSortPhysiNode *)pNode; - pPhysiChildren = groupSortPhysiNode->node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: { - SMergeIntervalPhysiNode *mergeIntPhysiNode = (SMergeIntervalPhysiNode *)pNode; - pPhysiChildren = mergeIntPhysiNode->window.node.pChildren; - break; - } - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: { - SInterpFuncPhysiNode *interpPhysiNode = (SInterpFuncPhysiNode *)pNode; - pPhysiChildren = interpPhysiNode->node.pChildren; - break; - } - default: - qError("not supported physical node type %d", pNode->type); - QRY_ERR_RET(TSDB_CODE_APP_ERROR); - } + SNodeList *pPhysiChildren = pNode->pChildren; if (pPhysiChildren) { *pChildren = nodesMakeList(); @@ -1583,6 +1461,36 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: { + SEventWinodwPhysiNode *pEventNode = (SEventWinodwPhysiNode *)pNode; + EXPLAIN_ROW_NEW(level, EXPLAIN_EVENT_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); + if (pResNode->pExecInfo) { + QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen)); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + } + EXPLAIN_ROW_APPEND(EXPLAIN_FUNCTIONS_FORMAT, pEventNode->window.pFuncs->length); + EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); + EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pEventNode->window.node.pOutputDataBlockDesc->totalRowSize); + EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); + + if (verbose) { + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_START_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pEventNode->pStartCond, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_EVENT_END_FORMAT); + QRY_ERR_RET(nodesNodeToSQL(pEventNode->pEndCond, tbuf + VARSTR_HEADER_SIZE, + TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); + EXPLAIN_ROW_END(); + QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); + } + break; + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_APP_ERROR; @@ -1666,7 +1574,6 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htobe64((int64_t)rowNum); int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); - ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 999a7965fb69b44cf80740e229cc546a38a32af6..1712cba0f53ac3e3f5cf69d3ad8e257851fd55f0 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -126,12 +126,14 @@ enum { typedef struct { // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - const SSubmitReq* pReq; + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + // const SSubmitReq* pReq; + + SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -355,6 +357,7 @@ typedef struct STableMergeScanInfo { SLimitInfo limitInfo; int64_t numOfRows; SScanInfo scanInfo; + int32_t scanTimes; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info SSortExecInfo sortExecInfo; @@ -471,11 +474,13 @@ typedef struct SStreamScanInfo { SNode* pTagIndexCond; // recover - int32_t blockRecoverContiCnt; - int32_t blockRecoverTotCnt; + int32_t blockRecoverContiCnt; + int32_t blockRecoverTotCnt; + SSDataBlock* pRecoverRes; - int8_t igCheckUpdate; - int8_t igExpired; + SSDataBlock* pCreateTbRes; + int8_t igCheckUpdate; + int8_t igExpired; } SStreamScanInfo; typedef struct { @@ -566,6 +571,8 @@ typedef struct SStreamIntervalOperatorInfo { SStreamState* pState; SWinKey delKey; uint64_t numOfDatapack; + SArray* pUpdated; + SSHashObj* pUpdatedMap; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -612,6 +619,8 @@ typedef struct SStreamSessionAggOperatorInfo { SPhysiNode* pPhyNode; // create new child bool isFinal; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pStUpdated; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -627,6 +636,8 @@ typedef struct SStreamStateAggOperatorInfo { void* pDelIterator; SArray* pChildren; // cache for children's result; bool ignoreExpiredData; + SArray* pUpdated; + SSHashObj* pSeUpdated; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -637,9 +648,11 @@ typedef struct SStreamPartitionOperatorInfo { SExprSupp tagCalSup; SHashObj* pPartitions; void* parIte; + void* pTbNameIte; SSDataBlock* pInputDataBlock; int32_t tsColIndex; SSDataBlock* pDelRes; + SSDataBlock* pCreateTbRes; } SStreamPartitionOperatorInfo; typedef struct SStreamFillSupporter { @@ -733,6 +746,7 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); +void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, @@ -763,7 +777,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); @@ -804,6 +818,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo); // clang-format on int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, @@ -839,7 +855,6 @@ bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pS void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock); int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); @@ -856,8 +871,17 @@ int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResul int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); -int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, - int32_t order, int64_t* pData); +int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, + int64_t* pData); +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); + +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); + +void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, + SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo); +void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset); #ifdef __cplusplus } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 2366d0c2f1bffd287f57414a9f484908c56b3113..dc3bf83a914eca1a93304b1487b7acdeca59c1ba 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -139,7 +139,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, taosFreeQitem(pBuf); return TSDB_CODE_OUT_OF_MEMORY; } - + toDataCacheEntry(pDeleter, pInput, pBuf); taosWriteQitem(pDeleter->pDataBlocks, pBuf); *pContinue = (DS_BUF_LOW == updateStatus(pDeleter) ? true : false); @@ -164,8 +164,10 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); - memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); - taosFreeQitem(pBuf); + if (pBuf != NULL) { + memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); + taosFreeQitem(pBuf); + } SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; *pLen = pEntry->dataLen; @@ -253,7 +255,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData deleter->pDeleter = pDeleterNode; deleter->pSchema = pDataSink->pInputDataBlockDesc; - if(pParam == NULL) { + if (pParam == NULL) { code = TSDB_CODE_QRY_INVALID_INPUT; qError("invalid input param in creating data deleter, code%s", tstrerror(code)); goto _end; @@ -272,7 +274,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData *pHandle = deleter; return code; - _end: +_end: if (deleter != NULL) { destroyDataSinker((SDataSinkHandle*)deleter); taosMemoryFree(deleter); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a603bffba5f37b79ee9f4a42546b8bbb3a0c32ea..d8efcf50cac24dca0b444f2e54bf886453e1bb96 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -77,8 +77,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); -// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); -// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); + // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); + // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); pBuf->useSize += pEntry->dataLen; @@ -135,7 +135,7 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, taosFreeQitem(pBuf); return TSDB_CODE_OUT_OF_MEMORY; } - + toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); @@ -162,14 +162,16 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); - memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); - taosFreeQitem(pBuf); + if (pBuf != NULL) { + memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); + taosFreeQitem(pBuf); + } SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; *pLen = pEntry->dataLen; -// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); -// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); + // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); + // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, @@ -192,8 +194,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; -// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); -// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); + // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); + // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 99d1968a5de525c4f1f44c2b3ebd89dbabbe4c0d..e9c46843c02cbe9c736dda90a0cf66fa22a780b4 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -25,9 +25,9 @@ extern SDataSinkStat gDataSinkStat; typedef struct SSubmitRes { - int64_t affectedRows; - int32_t code; - SSubmitRsp* pRsp; + int64_t affectedRows; + int32_t code; + SSubmitRsp2* pRsp; } SSubmitRes; typedef struct SDataInserterHandle { @@ -46,6 +46,7 @@ typedef struct SDataInserterHandle { uint64_t cachedSize; TdThreadMutex mutex; tsem_t ready; + bool explain; } SDataInserterHandle; typedef struct SSubmitRspParam { @@ -59,22 +60,25 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { pInserter->submitRes.code = code; if (code == TSDB_CODE_SUCCESS) { - pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp)); + pInserter->submitRes.pRsp = taosMemoryCalloc(1, sizeof(SSubmitRsp2)); SDecoder coder = {0}; tDecoderInit(&coder, pMsg->pData, pMsg->len); - code = tDecodeSSubmitRsp(&coder, pInserter->submitRes.pRsp); + code = tDecodeSSubmitRsp2(&coder, pInserter->submitRes.pRsp); if (code) { - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } - if (pInserter->submitRes.pRsp->nBlocks > 0) { - for (int32_t i = 0; i < pInserter->submitRes.pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pInserter->submitRes.pRsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + if (pInserter->submitRes.pRsp->affectedRows > 0) { + SArray* pCreateTbList = pInserter->submitRes.pRsp->aCreateTbRsp; + int32_t numOfTables = taosArrayGetSize(pCreateTbList); + + for (int32_t i = 0; i < numOfTables; ++i) { + SVCreateTbRsp* pRsp = taosArrayGet(pCreateTbList, i); + if (TSDB_CODE_SUCCESS != pRsp->code) { + code = pRsp->code; + taosMemoryFree(pInserter->submitRes.pRsp); pInserter->submitRes.code = code; goto _return; } @@ -82,22 +86,20 @@ int32_t inserterCallback(void* param, SDataBuf* pMsg, int32_t code) { } pInserter->submitRes.affectedRows += pInserter->submitRes.pRsp->affectedRows; - qDebug("submit rsp received, affectedRows:%d, total:%"PRId64, pInserter->submitRes.pRsp->affectedRows, + qDebug("submit rsp received, affectedRows:%d, total:%" PRId64, pInserter->submitRes.pRsp->affectedRows, pInserter->submitRes.affectedRows); - - tFreeSSubmitRsp(pInserter->submitRes.pRsp); + tDecoderClear(&coder); + taosMemoryFree(pInserter->submitRes.pRsp); } _return: - tsem_post(&pInserter->ready); - taosMemoryFree(pMsg->pData); - return TSDB_CODE_SUCCESS; } -static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMsg, void* pTransporter, SEpSet* pEpset) { +static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, void* pMsg, int32_t msgLen, void* pTransporter, + SEpSet* pEpset) { // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { @@ -112,7 +114,7 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = ntohl(pMsg->length); + pMsgSendInfo->msgInfo.len = msgLen; pMsgSendInfo->msgType = TDMT_VND_SUBMIT; pMsgSendInfo->fp = inserterCallback; @@ -120,147 +122,249 @@ static int32_t sendSubmitRequest(SDataInserterHandle* pInserter, SSubmitReq* pMs return asyncSendMsgToServer(pTransporter, pEpset, &transporterId, pMsgSendInfo); } -int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) { - const SArray* pBlocks = pInserter->pDataBlocks; - const STSchema* pTSchema = pInserter->pSchema; - int64_t uid = pInserter->pNode->tableId; - int64_t suid = pInserter->pNode->stableId; - int32_t vgId = pInserter->pNode->vgId; +static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SSubmitReq2Msg); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId); + ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); + ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); + } - SSubmitReq* ret = NULL; - int32_t sz = taosArrayGetSize(pBlocks); + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; +} - // cal size - int32_t cap = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); +int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** ppReq, const SSDataBlock* pDataBlock, + const STSchema* pTSchema, int64_t uid, int32_t vgId, tb_uid_t suid) { + SSubmitReq2* pReq = *ppReq; + SArray* pVals = NULL; + int32_t numOfBlks = 0; + + terrno = TSDB_CODE_SUCCESS; + + if (NULL == pReq) { + if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } - cap += sizeof(SSubmitBlk) + rows * maxLen; + if (!(pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } } - // assign data - // TODO - ret = taosMemoryCalloc(1, cap); - ret->header.vgId = htonl(vgId); - ret->version = htonl(pTSchema->version); - ret->length = sizeof(SSubmitReq); - ret->numOfBlocks = htonl(sz); + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; - SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + SSubmitTbData tbData = {0}; + if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow*)))) { + goto _end; + } + tbData.suid = suid; + tbData.uid = uid; + tbData.sver = pTSchema->version; - blkHead->sversion = htonl(pTSchema->version); - // TODO - blkHead->suid = htobe64(suid); - blkHead->uid = htobe64(uid); - blkHead->schemaLen = htonl(0); - - int32_t rows = 0; - int32_t dataLen = 0; - STSRow* rowData = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); - int64_t lastTs = TSKEY_MIN; - bool ignoreRow = false; - for (int32_t j = 0; j < pDataBlock->info.rows; j++) { - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); - tdSRowResetBuf(&rb, rowData); + if (!pVals && !(pVals = taosArrayInit(colNum, sizeof(SColVal)))) { + taosArrayDestroy(tbData.aRowP); + goto _end; + } - ignoreRow = false; - for (int32_t k = 0; k < pTSchema->numOfCols; k++) { - const STColumn* pColumn = &pTSchema->columns[k]; - SColumnInfoData* pColData = NULL; - int16_t colIdx = k; - if (!pInserter->fullOrderColList) { - int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId)); - if (NULL == slotId) { - continue; - } + int64_t lastTs = TSKEY_MIN; + bool ignoreRow = false; + bool disorderTs = false; - colIdx = *slotId; - } + for (int32_t j = 0; j < rows; ++j) { // iterate by row + taosArrayClear(pVals); - pColData = taosArrayGet(pDataBlock->pDataBlock, colIdx); - if (pColData->info.type != pColumn->type) { - qError("col type mis-match, schema type:%d, type in block:%d", pColumn->type, pColData->info.type); - terrno = TSDB_CODE_APP_ERROR; - return TSDB_CODE_APP_ERROR; + int32_t offset = 0; + for (int32_t k = 0; k < pTSchema->numOfCols; ++k) { // iterate by column + int16_t colIdx = k; + const STColumn* pCol = &pTSchema->columns[k]; + if (!pInserter->fullOrderColList) { + int16_t* slotId = taosHashGet(pInserter->pCols, &pCol->colId, sizeof(pCol->colId)); + if (NULL == slotId) { + continue; } - if (colDataIsNull_s(pColData, j)) { - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - ignoreRow = true; - break; - } + colIdx = *slotId; + } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, pColumn->offset, k); - } else { - void* data = colDataGetData(pColData, j); - if (0 == k && TSDB_DATA_TYPE_TIMESTAMP == pColumn->type) { - if (*(int64_t*)data == lastTs) { - ignoreRow = true; - break; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, colIdx); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + ASSERT(pColInfoData->info.type == pCol->type); + if (colDataIsNull_s(pColInfoData, j)) { + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); + taosArrayPush(pVals, &cv); + } else { + void* data = colDataGetVarData(pColInfoData, j); + SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); + } + break; + } + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_MEDIUMBLOB: + qError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); + terrno = TSDB_CODE_APP_ERROR; + goto _end; + break; + default: + if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { + if (colDataIsNull_s(pColInfoData, j)) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + qError("NULL value for primary key"); + terrno = TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL; + goto _end; + } + + SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); // should use pCol->type + taosArrayPush(pVals, &cv); } else { - lastTs = *(int64_t*)data; + if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { + if (*(int64_t*)var == lastTs) { + ignoreRow = true; + } else if (*(int64_t*)var < lastTs) { + disorderTs = true; + } else { + lastTs = *(int64_t*)var; + } + } + + SValue sv; + memcpy(&sv.val, var, tDataTypes[pCol->type].bytes); + SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv); + taosArrayPush(pVals, &cv); } + } else { + uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + terrno = TSDB_CODE_APP_ERROR; + goto _end; } - tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); - } - } - if (!pInserter->fullOrderColList) { - rb.hasNone = true; + break; } - tdSRowEnd(&rb); if (ignoreRow) { - continue; + break; } - - rows++; - int32_t rowLen = TD_ROW_LEN(rowData); - rowData = POINTER_SHIFT(rowData, rowLen); - dataLen += rowLen; } - blkHead->dataLen = htonl(dataLen); - blkHead->numOfRows = htonl(rows); + if (ignoreRow) { + ignoreRow = false; + continue; + } - ret->length += sizeof(SSubmitBlk) + dataLen; - blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + dataLen); + SRow* pRow = NULL; + if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { + tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + goto _end; + } + taosArrayPush(tbData.aRowP, &pRow); } - ret->length = htonl(ret->length); + if (disorderTs) { + tRowSort(tbData.aRowP); + if ((terrno = tRowMerge(tbData.aRowP, (STSchema*)pTSchema, 0)) != 0) { + goto _end; + } + } - *pReq = ret; + taosArrayPush(pReq->aSubmitTbData, &tbData); +_end: + taosArrayDestroy(pVals); + if (terrno != 0) { + *ppReq = NULL; + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + return terrno; + } + *ppReq = pReq; return TSDB_CODE_SUCCESS; } +int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32_t* msgLen) { + const SArray* pBlocks = pInserter->pDataBlocks; + const STSchema* pTSchema = pInserter->pSchema; + int64_t uid = pInserter->pNode->tableId; + int64_t suid = pInserter->pNode->stableId; + int32_t vgId = pInserter->pNode->vgId; + int32_t sz = taosArrayGetSize(pBlocks); + int32_t code = 0; + SSubmitReq2* pReq = NULL; + + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGetP(pBlocks, i); + + code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); + if (code) { + if (pReq) { + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + } + + return code; + } + } + + code = submitReqToMsg(vgId, pReq, pMsg, msgLen); + tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pReq); + + return code; +} + static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataInserterHandle* pInserter = (SDataInserterHandle*)pHandle; - taosArrayPush(pInserter->pDataBlocks, &pInput->pData); - SSubmitReq* pMsg = NULL; - int32_t code = dataBlockToSubmit(pInserter, &pMsg); - if (code) { - return code; - } + if (!pInserter->explain) { + taosArrayPush(pInserter->pDataBlocks, &pInput->pData); + void* pMsg = NULL; + int32_t msgLen = 0; + int32_t code = dataBlocksToSubmitReq(pInserter, &pMsg, &msgLen); + if (code) { + return code; + } - taosArrayClear(pInserter->pDataBlocks); - - code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); - if (code) { - return code; - } + taosArrayClear(pInserter->pDataBlocks); - tsem_wait(&pInserter->ready); + code = sendSubmitRequest(pInserter, pMsg, msgLen, pInserter->pParam->readHandle->pMsgCb->clientRpc, + &pInserter->pNode->epSet); + if (code) { + return code; + } - if (pInserter->submitRes.code) { - return pInserter->submitRes.code; + tsem_wait(&pInserter->ready); + + if (pInserter->submitRes.code) { + return pInserter->submitRes.code; + } } *pContinue = true; @@ -320,6 +424,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->pParam = pParam; inserter->status = DS_BUF_EMPTY; inserter->queryEnd = false; + inserter->explain = pInserterNode->explain; int64_t suid = 0; int32_t code = @@ -350,7 +455,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); - SNode* pNode = NULL; + SNode* pNode = NULL; int32_t i = 0; FOREACH(pNode, pInserterNode->pCols) { SColumnNode* pCol = (SColumnNode*)pNode; diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c new file mode 100644 index 0000000000000000000000000000000000000000..49e2d5bc4ac0c85fabd8ffde93aa527bbe3c85e0 --- /dev/null +++ b/source/libs/executor/src/eventwindowoperator.c @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executorimpl.h" +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "tcommon.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "ttime.h" + +typedef struct SEventWindowOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SExprSupp scalarSup; + SWindowRowsSup winSup; + int32_t tsSlotId; // primary timestamp column slot id + STimeWindowAggSupp twAggSup; + uint64_t groupId; // current group id, used to identify the data block from different groups + SFilterInfo* pStartCondInfo; + SFilterInfo* pEndCondInfo; + bool inWindow; + SResultRow* pRow; +} SEventWindowOperatorInfo; + +static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator); +static void destroyEWindowOperatorInfo(void* param); +static int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock); + +// todo : move to util +static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex, + uint64_t groupId) { + pRowSup->startRowIndex = rowIndex; + pRowSup->numOfRows = 0; + pRowSup->win.skey = tsList[rowIndex]; + pRowSup->groupId = groupId; +} + +static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) { + pRowSup->win.ekey = ts; + pRowSup->prevTs = ts; + pRowSup->numOfRows += 1; + pRowSup->groupId = groupId; +} + +static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { + int64_t* ts = (int64_t*)pColData->pData; + int32_t delta = includeEndpoint ? 1 : 0; + + int64_t duration = pWin->ekey - pWin->skey + delta; + ts[2] = duration; // set the duration + ts[3] = pWin->skey; // window start key + ts[4] = pWin->ekey + delta; // window end key +} + +SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, + SExecTaskInfo* pTaskInfo) { + SEventWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SEventWindowOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SEventWinodwPhysiNode* pEventWindowNode = (SEventWinodwPhysiNode*)physiNode; + + int32_t tsSlotId = ((SColumnNode*)pEventWindowNode->window.pTspk)->slotId; + int32_t code = filterInitFromNode((SNode*)pEventWindowNode->pStartCond, &pInfo->pStartCondInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pEventWindowNode->pEndCond, &pInfo->pEndCondInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + if (pEventWindowNode->window.pExprs != NULL) { + int32_t numOfScalarExpr = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + code = filterInitFromNode((SNode*)pEventWindowNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + int32_t num = 0; + SExprInfo* pExprInfo = createExprInfo(pEventWindowNode->window.pFuncs, NULL, &num); + initResultSizeInfo(&pOperator->resultInfo, 4096); + + code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, + pTaskInfo->streamInfo.pState); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + SSDataBlock* pResBlock = createDataBlockFromDescNode(pEventWindowNode->window.node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); + + initBasicInfo(&pInfo->binfo, pResBlock); + initResultRowInfo(&pInfo->binfo.resultRowInfo); + + pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pEventWindowNode->window.watermark, + .calTrigger = pEventWindowNode->window.triggerType}; + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->tsSlotId = tsSlotId; + + setOperatorInfo(pOperator, "EventWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo, + pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, eventWindowAggregate, NULL, destroyEWindowOperatorInfo, + optrDefaultBufFn, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + if (pInfo != NULL) { + destroyEWindowOperatorInfo(pInfo); + } + + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroyEWindowOperatorInfo(void* param) { + SEventWindowOperatorInfo* pInfo = (SEventWindowOperatorInfo*)param; + if (pInfo == NULL) { + return; + } + + if (pInfo->pRow != NULL) { + taosMemoryFree(pInfo->pRow); + } + + if (pInfo->pStartCondInfo != NULL) { + filterFreeInfo(pInfo->pStartCondInfo); + pInfo->pStartCondInfo = NULL; + } + + if (pInfo->pEndCondInfo != NULL) { + filterFreeInfo(pInfo->pEndCondInfo); + pInfo->pEndCondInfo = NULL; + } + + cleanupBasicInfo(&pInfo->binfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); + + cleanupAggSup(&pInfo->aggSup); + taosMemoryFreeClear(param); +} + +static SSDataBlock* eventWindowAggregate(SOperatorInfo* pOperator) { + SEventWindowOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SExprSupp* pSup = &pOperator->exprSupp; + int32_t order = TSDB_ORDER_ASC; + + SSDataBlock* pRes = pInfo->binfo.pRes; + + blockDataCleanup(pRes); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + + setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true); + blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId); + + // there is an scalar expression that needs to be calculated right before apply the group aggregation. + if (pInfo->scalarSup.pExprInfo != NULL) { + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); + } + } + + eventWindowAggImpl(pOperator, pInfo, pBlock); + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + return pRes; + } + } + + return pRes->info.rows == 0 ? NULL : pRes; +} + +static int32_t setSingleOutputTupleBufv1(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult, + SExprSupp* pExprSup, SAggSupporter* pAggSup) { + if (*pResult == NULL) { + SResultRow* p = taosMemoryCalloc(1, pAggSup->resultRowSize); + pResultRowInfo->cur = (SResultRowPosition){.pageId = p->pageId, .offset = p->offset}; + *pResult = p; + } + + (*pResult)->win = *win; + + clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs); + setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); + return TSDB_CODE_SUCCESS; +} + +static void doEventWindowAggImpl(SEventWindowOperatorInfo* pInfo, SExprSupp* pSup, int32_t startIndex, int32_t endIndex, + const SSDataBlock* pBlock, int64_t* tsList, SExecTaskInfo* pTaskInfo) { + SWindowRowsSup* pRowSup = &pInfo->winSup; + + int32_t numOfOutput = pSup->numOfExprs; + int32_t numOfRows = endIndex - startIndex + 1; + + doKeepTuple(pRowSup, tsList[endIndex], pBlock->info.id.groupId); + + int32_t ret = + setSingleOutputTupleBufv1(&pInfo->binfo.resultRowInfo, &pRowSup->win, &pInfo->pRow, pSup, &pInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); + } + + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); + applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startIndex, numOfRows, + pBlock->info.rows, numOfOutput); +} + +int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pRes = pInfo->binfo.pRes; + int64_t gid = pBlock->info.id.groupId; + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); + TSKEY* tsList = (TSKEY*)pColInfoData->pData; + SWindowRowsSup* pRowSup = &pInfo->winSup; + SColumnInfoData *ps = NULL, *pe = NULL; + int32_t rowIndex = 0; + + pRowSup->numOfRows = 0; + if (pInfo->groupId == 0) { + pInfo->groupId = gid; + } else if (pInfo->groupId != gid) { + // this is a new group, reset the info + pInfo->inWindow = false; + } + + SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; + + int32_t code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶m1); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t status1 = 0; + filterExecute(pInfo->pStartCondInfo, pBlock, &ps, NULL, param1.numOfCols, &status1); + + SFilterColumnParam param2 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock}; + code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶m2); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t status2 = 0; + filterExecute(pInfo->pEndCondInfo, pBlock, &pe, NULL, param2.numOfCols, &status2); + + int32_t startIndex = pInfo->inWindow ? 0 : -1; + while (rowIndex < pBlock->info.rows) { + if (pInfo->inWindow) { // let's find the first end value + for (rowIndex = startIndex; rowIndex < pBlock->info.rows; ++rowIndex) { + if (((bool*)pe->pData)[rowIndex]) { + break; + } + } + + if (rowIndex < pBlock->info.rows) { + doEventWindowAggImpl(pInfo, pSup, startIndex, rowIndex, pBlock, tsList, pTaskInfo); + doUpdateNumOfRows(pSup->pCtx, pInfo->pRow, pSup->numOfExprs, pSup->rowEntryInfoOffset); + + // check buffer size + if (pRes->info.rows + pInfo->pRow->numOfRows >= pRes->info.capacity) { + int32_t newSize = pRes->info.rows + pInfo->pRow->numOfRows; + blockDataEnsureCapacity(pRes, newSize); + } + + copyResultrowToDataBlock(pSup->pExprInfo, pSup->numOfExprs, pInfo->pRow, pSup->pCtx, pRes, + pSup->rowEntryInfoOffset, pTaskInfo); + + pRes->info.rows += pInfo->pRow->numOfRows; + + pInfo->inWindow = false; + rowIndex += 1; + } else { + doEventWindowAggImpl(pInfo, pSup, startIndex, pBlock->info.rows - 1, pBlock, tsList, pTaskInfo); + } + } else { // find the first start value that is fulfill for the start condition + for (; rowIndex < pBlock->info.rows; ++rowIndex) { + if (((bool*)ps->pData)[rowIndex]) { + doKeepNewWindowStartInfo(pRowSup, tsList, rowIndex, gid); + pInfo->inWindow = true; + startIndex = rowIndex; + break; + } + } + + if (pInfo->inWindow) { + continue; // try to find the end position + } else { + break; // no valid start position, quit + } + } + } + + colDataDestroy(ps); + taosMemoryFree(ps); + colDataDestroy(pe); + taosMemoryFree(pe); + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a04da867e2a63b7f14d29da12d1e5bc3244205b0..6b6f5cfe93a3e3ffd89c9f277d41a5dd4ee92743 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -43,12 +43,20 @@ typedef struct tagFilterAssist { SArray* cInfoList; } tagFilterAssist; -static int32_t removeInvalidUid(SArray* uids, SHashObj* tags); -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* pTagCond); -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond); -static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle); +typedef enum { + FILTER_NO_LOGIC = 1, + FILTER_AND, + FILTER_OTHER, +} FilterCondType; + +static FilterCondType checkTagCond(SNode* cond); +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); + +static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, + SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, + void* metaHandle); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -89,14 +97,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return rowSize; } -static void freeEx(void* p) { - taosMemoryFree(*(void**)p); -} +static void freeEx(void* p) { taosMemoryFree(*(void**)p); } void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { taosMemoryFreeClear(pGroupResInfo->pBuf); if (pGroupResInfo->freeItem) { -// taosArrayDestroy(pGroupResInfo->pRows); + // taosArrayDestroy(pGroupResInfo->pRows); taosArrayDestroyEx(pGroupResInfo->pRows, freeEx); pGroupResInfo->freeItem = false; pGroupResInfo->pRows = NULL; @@ -143,7 +149,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { - /*void* key = */tSimpleHashGetKey(pData, &keyLen); + /*void* key = */ tSimpleHashGetKey(pData, &keyLen); bufLen += keyLen + sizeof(SResultRowPosition); } @@ -153,7 +159,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { void* key = tSimpleHashGetKey(pData, &keyLen); - SResKeyPos* p = (SResKeyPos*) (pGroupResInfo->pBuf + offset); + SResKeyPos* p = (SResKeyPos*)(pGroupResInfo->pBuf + offset); p->groupId = *(uint64_t*)key; p->pos = *(SResultRowPosition*)pData; @@ -174,7 +180,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { if (pGroupResInfo->pRows != NULL) { - taosArrayDestroyP(pGroupResInfo->pRows, taosMemoryFree); + taosArrayDestroy(pGroupResInfo->pRows); } pGroupResInfo->freeItem = true; @@ -417,7 +423,7 @@ static void releaseColInfoData(void* pCol) { } void freeItem(void* p) { - STUidTagInfo *pInfo = p; + STUidTagInfo* pInfo = p; if (pInfo->pTagVal != NULL) { taosMemoryFree(pInfo->pTagVal); } @@ -458,7 +464,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); - STUidTagInfo info = {.uid = pkeyInfo->uid}; + STUidTagInfo info = {.uid = pkeyInfo->uid}; taosArrayPush(pUidTagList, &info); } @@ -656,22 +662,32 @@ static int tableUidCompare(const void* a, const void* b) { } static int32_t filterTableInfoCompare(const void* a, const void* b) { - STUidTagInfo* p1 = (STUidTagInfo*) a; - STUidTagInfo* p2 = (STUidTagInfo*) b; + STUidTagInfo* p1 = (STUidTagInfo*)a; + STUidTagInfo* p2 = (STUidTagInfo*)b; if (p1->uid == p2->uid) { return 0; } - return p1->uid < p2->uid? -1:1; + return p1->uid < p2->uid ? -1 : 1; } -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* cond) { +static FilterCondType checkTagCond(SNode* cond) { + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + return FILTER_NO_LOGIC; + } + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return FILTER_AND; + } + return FILTER_OTHER; +} + +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { int32_t ret = -1; int32_t ntype = nodeType(cond); if (ntype == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, pRes, cond); + ret = optimizeTbnameInCondImpl(metaHandle, list, cond); } if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { @@ -690,50 +706,23 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, pRes, cell->pNode) == 0) { + if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { hasTbnameCond = true; break; } cell = cell->pNext; } - taosArraySort(pRes, filterTableInfoCompare); - taosArrayRemoveDuplicate(pRes, filterTableInfoCompare, NULL); + taosArraySort(list, filterTableInfoCompare); + taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, pRes); -// removeInvalidUid(pRes, tags); + ret = metaGetTableTagsByUids(metaHandle, suid, list); } return ret; } -#if 0 -/* - * handle invalid uid - */ -static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) { - int32_t size = taosArrayGetSize(uids); - if (size <= 0) { - return 0; - } - - SArray* validUid = taosArrayInit(size, sizeof(STUidTagInfo)); - - for (int32_t i = 0; i < size; i++) { - STUidTagInfo* p = taosArrayGet(uids, i); - if (taosHashGet(tags, &p->uid, sizeof(int64_t)) != NULL) { - taosArrayPush(validUid, p); - } - } - - taosArraySwap(uids, validUid); - taosArrayDestroy(validUid); - return 0; -} - -#endif - // only return uid that does not contained in pExistedUidList static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { @@ -759,7 +748,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis int32_t numOfTables = taosArrayGetSize(pTbList); SHashObj* uHash = NULL; - size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids + size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids if (numOfExisted > 0) { uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfExisted; i++) { @@ -785,7 +774,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } } else { -// qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno)); + // qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno)); terrno = 0; } } @@ -814,7 +803,8 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, + void* metaHandle) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -847,7 +837,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; if (p1->name != NULL) { STR_TO_VARSTR(str, p1->name); - } else { // name is not retrieved during filter + } else { // name is not retrieved during filter metaGetTableNameByUid(metaHandle, p1->uid, str); } @@ -898,7 +888,7 @@ static void doSetQualifiedUid(SArray* pUidList, const SArray* pUidTagList, bool* taosArrayClear(pUidList); int32_t numOfTables = taosArrayGetSize(pUidTagList); - for(int32_t i = 0; i < numOfTables; ++i) { + for (int32_t i = 0; i < numOfTables; ++i) { uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid; qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResultList[i]); @@ -914,14 +904,15 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { return; } - for(int32_t i = 0; i < numOfExisted; ++i) { - uint64_t* uid = taosArrayGet(pUidList, i); + for (int32_t i = 0; i < numOfExisted; ++i) { + uint64_t* uid = taosArrayGet(pUidList, i); STUidTagInfo info = {.uid = *uid}; taosArrayPush(pUidTagList, &info); } } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, + SIdxFltStatus status) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } @@ -954,13 +945,15 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); copyExistedUids(pUidTagList, pUidList); + FilterCondType condType = checkTagCond(pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); int32_t numOfRows = taosArrayGetSize(pUidTagList); taosArrayEnsureCap(pUidList, numOfRows); - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i); taosArrayPush(pUidList, &pInfo->uid); } @@ -968,8 +961,11 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; goto end; } else { - // here we retrieve all tags from the vnode table-meta store - code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { + code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList); + } else { + code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); terrno = code; @@ -1006,9 +1002,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - doSetQualifiedUid(pUidList, pUidTagList, (bool*) output.columnData->pData); + doSetQualifiedUid(pUidList, pUidTagList, (bool*)output.columnData->pData); - end: +end: taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); @@ -1028,12 +1024,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); + SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (metaIsTableExist(metaHandle, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1045,7 +1042,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, genTagFilterDigest(pTagCond, &context); bool acquired = false; - metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired); + metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, + &acquired); if (acquired) { qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pUidList)); goto _end; @@ -1057,20 +1055,22 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = tsdbGetIvtIdx(metaHandle); + void* pIndex = tsdbGetIvtIdx(metaHandle); SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake -// qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); + // qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); code = TDB_CODE_SUCCESS; + } else { + qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); } } } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1102,7 +1102,7 @@ _end: return TSDB_CODE_OUT_OF_MEMORY; } - qTrace("tagfilter get uid:%" PRIu64", %s", info.uid, idstr); + qTrace("tagfilter get uid:%" PRIu64 ", %s", info.uid, idstr); } taosArrayDestroy(pUidList); @@ -1421,6 +1421,18 @@ void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { createExprFromOneNode(pExp, pTargetNode->pExpr, pTargetNode->slotId); } +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs) { + *numOfExprs = LIST_LENGTH(pNodeList); + SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); + + for (int32_t i = 0; i < (*numOfExprs); ++i) { + SExprInfo* pExp = &pExprs[i]; + createExprFromOneNode(pExp, nodesListGetNode(pNodeList, i), i + UD_TAG_COLUMN_INDEX); + } + + return pExprs; +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; @@ -1512,7 +1524,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); } else { char* udfName = pExpr->pExpr->_function.pFunctNode->functionName; - pCtx->udfName = strdup(udfName); + pCtx->udfName = taosStrdup(udfName); fmGetUdafExecFuncs(pCtx->functionId, &pCtx->fpSet); } pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); @@ -1817,7 +1829,7 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, int32_t* size) { int32_t totalGroups = tableListGetOutputGroups(pTableList); - int32_t numOfTables = tableListGetSize(pTableList); + int32_t numOfTables = tableListGetSize(pTableList); if (ordinalGroupIndex < 0 || ordinalGroupIndex >= totalGroups) { return TSDB_CODE_INVALID_PARA; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 82f079e2fb283732c71e0c16604e991568ce59ed..347ac369d883deae5104495e033f8c191962f032 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -54,8 +54,8 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf if (type == STREAM_INPUT__MERGED_SUBMIT) { for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { @@ -64,7 +64,10 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } @@ -114,18 +117,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { - SSubmitReq* pReq = *(void**)POINTER_SHIFT(input, i * sizeof(void*)); - taosArrayPush(pInfo->pBlockLists, &pReq); + SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); + taosArrayPush(pInfo->pBlockLists, pReq); } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(numOfBlocks == 1); - taosArrayPush(pInfo->pBlockLists, &input); + taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - taosArrayPush(pInfo->pBlockLists, &pDataBlock); + SPackedData tmp = { + .pDataBlock = pDataBlock, + }; + taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; } else { @@ -1016,11 +1022,22 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq) { +#if 0 +int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t scanVer) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); ASSERT(pTaskInfo->streamInfo.pReq == NULL); pTaskInfo->streamInfo.pReq = pReq; + pTaskInfo->streamInfo.scanVer = scanVer; + return 0; +} +#endif + +int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE); + ASSERT(pTaskInfo->streamInfo.submit.msgStr == NULL); + pTaskInfo->streamInfo.submit = submit; return 0; } @@ -1191,3 +1208,4 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); } + diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index cfbbaf2e7ee227a6d7fd3dfb096eec5c2bea721c..89e0dd363c714516fe14cbb0eab90c44015551bc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -177,7 +177,7 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i // set the number of rows in current disk page SResultRow* pResultRow = (SResultRow*)((char*)pData + pData->num); - memset((char*) pResultRow, 0, interBufSize); + memset((char*)pResultRow, 0, interBufSize); pResultRow->pageId = pageId; pResultRow->offset = (int32_t)pData->num; @@ -395,19 +395,20 @@ void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pC } } -static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlockInfo(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag) { SqlFunctionCtx* pCtx = pExprSup->pCtx; for (int32_t i = 0; i < pExprSup->numOfExprs; ++i) { pCtx[i].order = order; pCtx[i].input.numOfRows = pBlock->info.rows; setBlockSMAInfo(&pCtx[i], &pExprSup->pExprInfo[i], pBlock); pCtx[i].pSrcBlock = pBlock; + pCtx[i].scanFlag = scanFlag; } } void setInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { - doSetInputDataBlockInfo(pExprSup, pBlock, order); + doSetInputDataBlockInfo(pExprSup, pBlock, order, scanFlag); } else { doSetInputDataBlock(pExprSup, pBlock, order, scanFlag, createDummyCol); } @@ -536,7 +537,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return false; } - if (pCtx->scanFlag == REPEAT_SCAN) { + if (pCtx->scanFlag == PRE_SCAN) { return fmIsRepeatScanFunc(pCtx->functionId); } @@ -832,6 +833,20 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO } } +void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput) { + for (int32_t i = 0; i < numOfOutput; ++i) { + SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (pResInfo == NULL) { + continue; + } + + pResInfo->initialized = false; + pResInfo->numOfRes = 0; + pResInfo->isNullRes = 0; + pResInfo->complete = false; + } +} + void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo) { if (pFilterInfo == NULL || pBlock->info.rows == 0) { return; @@ -891,12 +906,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoD } int32_t numOfRows = 0; - if (IS_VAR_DATA_TYPE(pDst->info.type)) { int32_t j = 0; pDst->varmeta.length = 0; - while(j < totalRows) { + while (j < totalRows) { if (pIndicator[j] == 0) { j += 1; continue; @@ -1047,8 +1061,7 @@ static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, u pAggInfo->groupId = groupId; } -static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, - const int32_t* rowEntryOffset) { +void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset) { bool returnNotNull = false; for (int32_t j = 0; j < numOfExprs; ++j) { SResultRowEntryInfo* pResInfo = getResultEntryInfo(pRow, j, rowEntryOffset); @@ -1071,8 +1084,8 @@ static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t nu } } -static void doCopyResultToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, - SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo) { +void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, + SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo) { for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; @@ -1108,7 +1121,7 @@ static void doCopyResultToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SR // todo refactor. SResultRow has direct pointer in miainfo int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { - SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); if (page == NULL) { qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, terrno); @@ -1138,7 +1151,7 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos T_LONG_JMP(pTaskInfo->env, code); } - doCopyResultToDataBlock(pExprInfo, pSup->numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); + copyResultrowToDataBlock(pExprInfo, pSup->numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; @@ -1197,7 +1210,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } pGroupResInfo->index += 1; - doCopyResultToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); + copyResultrowToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; @@ -1718,8 +1731,8 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey) { - int32_t code = 0; -// _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + int32_t code = 0; + // _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pAggSup->currentPageId = -1; pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); @@ -1924,9 +1937,7 @@ _error: return NULL; } -void cleanupBasicInfo(SOptrBasicInfo* pInfo) { - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} +void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void freeItem(void* pItem) { void** p = pItem; @@ -1970,7 +1981,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->schemaInfo.dbname = strdup(dbFName); + pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); pTaskInfo->execModel = model; pTaskInfo->pTableInfoList = tableListCreate(); pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); @@ -1996,7 +2007,7 @@ int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, } SSchemaInfo* pSchemaInfo = &pTaskInfo->schemaInfo; - pSchemaInfo->tablename = strdup(mr.me.name); + pSchemaInfo->tablename = taosStrdup(mr.me.name); if (mr.me.type == TSDB_SUPER_TABLE) { pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); @@ -2274,9 +2285,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - - bool isStream = (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type); - pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo, isStream); + pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { @@ -2328,6 +2337,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) { + pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { terrno = TSDB_CODE_INVALID_PARA; return NULL; @@ -2595,26 +2606,22 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { - SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); - int32_t size = 0; - void* pVal = NULL; - SWinKey key = { - .ts = *(TSKEY*)pPos->key, - .groupId = pPos->groupId, - }; - int32_t code = streamStateGet(pState, &key, &pVal, &size); + SWinKey* pKey = taosArrayGet(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + int32_t code = streamStateGet(pState, pKey, &pVal, &size); ASSERT(code == 0); SResultRow* pRow = (SResultRow*)pVal; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pState, &key, pRow); + releaseOutputBuf(pState, pKey, pRow); continue; } if (pBlock->info.id.groupId == 0) { - pBlock->info.id.groupId = pPos->groupId; + pBlock->info.id.groupId = pKey->groupId; void* tbname = NULL; if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { pBlock->info.parTbName[0] = 0; @@ -2624,15 +2631,15 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat tdbFree(tbname); } else { // current value belongs to different group, it can't be packed into one datablock - if (pBlock->info.id.groupId != pPos->groupId) { - releaseOutputBuf(pState, &key, pRow); + if (pBlock->info.id.groupId != pKey->groupId) { + releaseOutputBuf(pState, pKey, pRow); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pState, &key, pRow); + releaseOutputBuf(pState, pKey, pRow); break; } @@ -2662,7 +2669,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat } pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pState, &key, pRow); + releaseOutputBuf(pState, pKey, pRow); } pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); @@ -2762,3 +2769,24 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; } + +void qStreamCloseTsdbReader(void* task) { + if (task == NULL) return; + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; + SOperatorInfo* pOp = pTaskInfo->pRoot; + qDebug("stream close tsdb reader, reset status uid %" PRId64 " ts %" PRId64, pTaskInfo->streamInfo.lastStatus.uid, + pTaskInfo->streamInfo.lastStatus.ts); + pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; + while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { + SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; + if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pDownstreamOp->info; + if (pInfo->pTableScanOp) { + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + return; + } + } + } +} diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index a1426e2a969d9b2f2f5a866efbd63bd6f057ca17..adb045a69fa874e6f7beddd63eadb0199ddb9bfe 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -1265,9 +1265,11 @@ static SSDataBlock* doStreamFill(SOperatorInfo* pOperator) { memcpy(pInfo->pSrcBlock->info.parTbName, pBlock->info.parTbName, TSDB_TABLE_NAME_LEN); pInfo->srcRowIndex = 0; } break; + case STREAM_CREATE_CHILD_TABLE: { + return pBlock; + } break; default: - ASSERT(0); - break; + ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index aa61d24b92630a48ca7f284912d82d8b45e2490f..8f5c3786e0c64d027f3e1bec4c2880a11016aa4c 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -426,9 +426,11 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { } SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; SGroupbyOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupbyOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -442,7 +444,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* } pInfo->pGroupCols = extractColumnInfo(pAggNode->pGroupKeys); - int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -481,7 +483,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* return pOperator; _error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; if (pInfo != NULL) { destroyGroupOperatorInfo(pInfo); } @@ -937,6 +939,7 @@ uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, S } static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } +static bool hasRemainTbName(SStreamPartitionOperatorInfo* pInfo) { return pInfo->pTbNameIte != NULL; } static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { SStreamPartitionOperatorInfo* pInfo = pOperator->info; @@ -957,40 +960,13 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { colDataSetVal(pDestCol, pDest->info.rows, pSrcData, isNull); } pDest->info.rows++; - if (pInfo->tbnameCalSup.numOfExprs > 0 && i == 0) { - void* tbname = NULL; - if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { - memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - tdbFree(tbname); - } else { - SSDataBlock* pTmpBlock = blockCopyOneRow(pSrc, rowIndex); - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - projectApplyFunctions(pInfo->tbnameCalSup.pExprInfo, pResBlock, pTmpBlock, pInfo->tbnameCalSup.pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - void* pData = colDataGetVarData(pCol, 0); - // TODO check tbname validity - if (pData != (void*)-1) { - memset(pDest->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pDest->info.parTbName, varDataVal(pData), len); - /*pDest->info.parTbName[len + 1] = 0;*/ - } else { - pDest->info.parTbName[0] = 0; - } - if (pParInfo->groupId && pDest->info.parTbName[0]) { - streamStatePutParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, pDest->info.parTbName); - } - /*printf("\n\n set name %s\n\n", pDest->info.parTbName);*/ - blockDataDestroy(pTmpBlock); - blockDataDestroy(pResBlock); - } + } + pDest->info.parTbName[0] = 0; + if (pInfo->tbnameCalSup.numOfExprs > 0) { + void* tbname = NULL; + if (streamStateGetParName(pOperator->pTaskInfo->streamInfo.pState, pParInfo->groupId, &tbname) == 0) { + memcpy(pDest->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); + tdbFree(tbname); } } taosArrayDestroy(pParInfo->rowIds); @@ -1006,6 +982,73 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { return pDest; } +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock) { + void* pValue = NULL; + if (streamStateGetParName(pState, groupId, &pValue) != 0) { + SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId); + memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); + pTmpBlock->info.id.groupId = groupId; + char* tbName = pSrcBlock->info.parTbName; + if (pTableSup->numOfExprs > 0) { + projectApplyFunctions(pTableSup->pExprInfo, pDestBlock, pTmpBlock, pTableSup->pCtx, pTableSup->numOfExprs, NULL); + SColumnInfoData* pTbCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + memset(tbName, 0, TSDB_TABLE_NAME_LEN); + int32_t len = 0; + if (colDataIsNull_s(pTbCol, pDestBlock->info.rows - 1)) { + len = TMIN(sizeof(TSDB_DATA_NULL_STR), TSDB_TABLE_NAME_LEN - 1); + memcpy(tbName, TSDB_DATA_NULL_STR, len); + } else { + void* pData = colDataGetData(pTbCol, pDestBlock->info.rows - 1); + len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); + memcpy(tbName, varDataVal(pData), len); + } + streamStatePutParName(pState, groupId, tbName); + memcpy(pTmpBlock->info.parTbName, tbName, len); + pDestBlock->info.rows--; + } else { + void* pTbNameCol = taosArrayGet(pDestBlock->pDataBlock, UD_TABLE_NAME_COLUMN_INDEX); + colDataSetNULL(pTbNameCol, pDestBlock->info.rows); + tbName[0] = 0; + } + + if (pTagSup->numOfExprs > 0) { + projectApplyFunctions(pTagSup->pExprInfo, pDestBlock, pTmpBlock, pTagSup->pCtx, pTagSup->numOfExprs, NULL); + pDestBlock->info.rows--; + } else { + memcpy(pDestBlock->info.parTbName, pTmpBlock->info.parTbName, TSDB_TABLE_NAME_LEN); + } + + void* pGpIdCol = taosArrayGet(pDestBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); + colDataAppend(pGpIdCol, pDestBlock->info.rows, (const char*)&groupId, false); + pDestBlock->info.rows++; + blockDataDestroy(pTmpBlock); + } else { + memcpy(pSrcBlock->info.parTbName, pValue, TSDB_TABLE_NAME_LEN); + } + streamStateReleaseBuf(pState, NULL, pValue); +} + +static SSDataBlock* buildStreamCreateTableResult(SOperatorInfo* pOperator) { + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if ((pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) || + taosHashGetSize(pInfo->pPartitions) == 0) { + return NULL; + } + blockDataCleanup(pInfo->pCreateTbRes); + blockDataEnsureCapacity(pInfo->pCreateTbRes, taosHashGetSize(pInfo->pPartitions)); + SSDataBlock* pSrc = pInfo->pInputDataBlock; + + if (pInfo->pTbNameIte != NULL) { + SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->pTbNameIte; + int32_t rowId = *(int32_t*)taosArrayGet(pParInfo->rowIds, 0); + appendCreateTableRow(pOperator->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pParInfo->groupId, pSrc, rowId, pInfo->pCreateTbRes); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, pInfo->pTbNameIte); + } + return pInfo->pCreateTbRes->info.rows > 0 ? pInfo->pCreateTbRes : NULL; +} + static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { pInfo->pInputDataBlock = pBlock; for (int32_t i = 0; i < pBlock->info.rows; ++i) { @@ -1032,6 +1075,15 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pCtRes = NULL; + + if (hasRemainTbName(pInfo)) { + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } + } + if (hasRemainPartion(pInfo)) { return buildStreamPartitionResult(pOperator); } @@ -1059,6 +1111,7 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { return pInfo->pDelRes; } break; default: + ASSERTS(pBlock->info.type == STREAM_CREATE_CHILD_TABLE, "invalid SSDataBlock type"); return pBlock; } @@ -1073,6 +1126,11 @@ static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); + pInfo->pTbNameIte = taosHashIterate(pInfo->pPartitions, NULL); + pCtRes = buildStreamCreateTableResult(pOperator); + if (pCtRes != NULL) { + return pCtRes; + } return buildStreamPartitionResult(pOperator); } @@ -1093,6 +1151,7 @@ static void destroyStreamPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->tagCalSup); blockDataDestroy(pInfo->pDelRes); taosHashCleanup(pInfo->pPartitions); + blockDataDestroy(pInfo->pCreateTbRes); taosMemoryFreeClear(param); } @@ -1108,14 +1167,56 @@ void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup } } +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.id.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_CREATE_CHILD_TABLE; + pBlock->info.watermark = INT64_MIN; + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + if (tbName->numOfExprs > 0) { + infoData.info.bytes = tbName->pExprInfo->base.resSchema.bytes; + } else { + infoData.info.bytes = 1; + } + pBlock->info.rowSize += infoData.info.bytes; + // sub table name + taosArrayPush(pBlock->pDataBlock, &infoData); + + SColumnInfoData gpIdData = {0}; + gpIdData.info.type = TSDB_DATA_TYPE_UBIGINT; + gpIdData.info.bytes = 8; + pBlock->info.rowSize += gpIdData.info.bytes; + // group id + taosArrayPush(pBlock->pDataBlock, &gpIdData); + + for (int32_t i = 0; i < tag->numOfExprs; i++) { + SColumnInfoData tagCol = {0}; + tagCol.info.type = tag->pExprInfo[i].base.resSchema.type; + tagCol.info.bytes = tag->pExprInfo[i].base.resSchema.bytes; + tagCol.info.precision = tag->pExprInfo[i].base.resSchema.precision; + // tag info + taosArrayPush(pBlock->pDataBlock, &tagCol); + pBlock->info.rowSize += tagCol.info.bytes; + } + + return pBlock; +} + SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { + int32_t code = TSDB_CODE_SUCCESS; SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } - int32_t code = TSDB_CODE_SUCCESS; + pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->part.pPartitionKeys); if (pPartNode->part.pExprs != NULL) { @@ -1127,6 +1228,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tbnameCalSup.numOfExprs = 0; if (pPartNode->pSubtable != NULL) { SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo)); if (pSubTableExpr == NULL) { @@ -1141,19 +1243,26 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr } } + pInfo->tagCalSup.numOfExprs = 0; if (pPartNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pPartNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pPartNode->pTags, &numOfTags); if (pTagExpr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } if (initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags) != 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } } + if (pInfo->tbnameCalSup.numOfExprs != 0 || pInfo->tagCalSup.numOfExprs != 0) { + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + } else { + pInfo->pCreateTbRes = NULL; + } + int32_t keyLen = 0; code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupCols); @@ -1164,12 +1273,14 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->part.node.pOutputDataBlockDesc); if (pInfo->binfo.pRes == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } blockDataEnsureCapacity(pInfo->binfo.pRes, 4096); pInfo->parIte = NULL; + pInfo->pTbNameIte = NULL; pInfo->pInputDataBlock = NULL; _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -1192,7 +1303,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr return pOperator; _error: - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = code; destroyStreamPartitionOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); return NULL; diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 60b076e3949ecc07404a25fadc30fea50a465e4d..3ae114c6563f8af52075e243d00a5a2f75ba6167 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -218,7 +218,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.pReq) { + if (pTaskInfo->streamInfo.submit.msgStr) { pOperator->status = OP_OPENED; } @@ -274,7 +274,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || - pBlock->info.type == STREAM_DELETE_DATA) { + pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { return pBlock; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 71e1068fb39831d07040b61c99e7a19c7b3494db..a238b84993e5d3b07a31caba108e82ebd44a3f39 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -309,12 +309,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%"PRIu64, GET_TASKID(pTaskInfo), + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) { pCost->loadBlockStatis += 1; @@ -324,6 +326,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -345,6 +348,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; + tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } } @@ -359,7 +363,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; - + tsdbReleaseDataBlock(pTableScanInfo->dataReader); *status = FUNC_DATA_REQUIRED_FILTEROUT; return TSDB_CODE_SUCCESS; } @@ -395,7 +399,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca } bool limitReached = applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo); - if (limitReached) { // set operator flag is done + if (limitReached) { // set operator flag is done setOperatorCompleted(pOperator); } @@ -430,7 +434,7 @@ static void freeTableCachedVal(void* param) { static STableCachedVal* createTableCacheVal(const SMetaReader* pMetaReader) { STableCachedVal* pVal = taosMemoryMalloc(sizeof(STableCachedVal)); - pVal->pName = strdup(pMetaReader->me.name); + pVal->pName = taosStrdup(pMetaReader->me.name); pVal->pTags = NULL; // only child table has tag value @@ -446,7 +450,6 @@ static STableCachedVal* createTableCacheVal(const SMetaReader* pMetaReader) { // const void *key, size_t keyLen, void *value static void freeCachedMetaItem(const void* key, size_t keyLen, void* value) { freeTableCachedVal(value); } - static void doSetNullValue(SSDataBlock* pBlock, const SExprInfo* pExpr, int32_t numOfExpr) { for (int32_t j = 0; j < numOfExpr; ++j) { int32_t dstSlotId = pExpr[j].base.resSchema.slotId; @@ -483,7 +486,6 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int metaReaderInit(&mr, pHandle->meta, 0); code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid); if (code != TSDB_CODE_SUCCESS) { - // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed. if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) { qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", @@ -634,10 +636,12 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { while (tsdbNextDataBlock(pTableScanInfo->base.dataReader)) { if (isTaskKilled(pTaskInfo)) { + tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } if (pOperator->status == OP_EXEC_DONE) { + tsdbReleaseDataBlock(pTableScanInfo->base.dataReader); break; } @@ -699,7 +703,8 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->base.scanFlag = REPEAT_SCAN; + pTableScanInfo->base.scanFlag = MAIN_SCAN; + pTableScanInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD; qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo)); // do prepare for the next round table scan operation @@ -725,7 +730,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->scanTimes < total) { setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTableScanInfo->base.scanFlag = REPEAT_SCAN; + pTableScanInfo->base.scanFlag = MAIN_SCAN; qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo)); tsdbReaderReset(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond); @@ -877,8 +882,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; + pInfo->base.scanFlag = (pInfo->scanInfo.numOfAsc > 1) ? PRE_SCAN : MAIN_SCAN; - pInfo->base.scanFlag = MAIN_SCAN; pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode); pInfo->base.readHandle = *readHandle; pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired; @@ -888,7 +893,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, initResultSizeInfo(&pOperator->resultInfo, 4096); pInfo->pResBlock = createDataBlockFromDescNode(pDescNode); -// blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); + // blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity); code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -1176,11 +1181,11 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 } static int32_t getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, - SSessionKey* pKey) { + SSessionKey* pKey) { pKey->win.skey = startTs; pKey->win.ekey = endTs; pKey->groupId = groupId; - + SStreamStateCur* pCur = streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey); int32_t code = streamStateSessionGetKVByCur(pCur, pKey, NULL, 0); if (code != TSDB_CODE_SUCCESS) { @@ -1224,12 +1229,12 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr } SSessionKey endWin = {0}; getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin); - if(IS_INVALID_SESSION_WIN_KEY(endWin)) { + if (IS_INVALID_SESSION_WIN_KEY(endWin)) { getPreSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin); } if (IS_INVALID_SESSION_WIN_KEY(startWin)) { // window has been closed. - qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64 , startData[i], endData[i]); + qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64, startData[i], endData[i]); continue; } colDataSetVal(pDestStartCol, i, (const char*)&startWin.win.skey, false); @@ -1379,52 +1384,46 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, return code; } -void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { - SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; +#if 0 +void calBlockTag(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + SExprSupp* pTagCalSup = &pInfo->tagCalSup; SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; - if (pTbNameCalSup == NULL || pTbNameCalSup->numOfExprs == 0) return; + if (pTagCalSup == NULL || pTagCalSup->numOfExprs == 0) return; if (pBlock == NULL || pBlock->info.rows == 0) return; - void* tbname = NULL; - if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) { - pBlock->info.parTbName[0] = 0; - } else { - memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN); - } - tdbFree(tbname); - - SSDataBlock* pSrcBlock = blockCopyOneRow(pBlock, 0); - ASSERT(pSrcBlock->info.rows == 1); - - SSDataBlock* pResBlock = createDataBlock(); - pResBlock->info.rowSize = VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN; - SColumnInfoData data = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_TABLE_NAME_LEN, 0); - taosArrayPush(pResBlock->pDataBlock, &data); - blockDataEnsureCapacity(pResBlock, 1); - - projectApplyFunctions(pTbNameCalSup->pExprInfo, pResBlock, pSrcBlock, pTbNameCalSup->pCtx, 1, NULL); - ASSERT(pResBlock->info.rows == 1); - ASSERT(taosArrayGetSize(pResBlock->pDataBlock) == 1); - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, 0); - ASSERT(pCol->info.type == TSDB_DATA_TYPE_VARCHAR); - - void* pData = colDataGetData(pCol, 0); - // TODO check tbname validation - if (pData != (void*)-1 && pData != NULL) { - memset(pBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN); - int32_t len = TMIN(varDataLen(pData), TSDB_TABLE_NAME_LEN - 1); - memcpy(pBlock->info.parTbName, varDataVal(pData), len); - /*pBlock->info.parTbName[len + 1] = 0;*/ + void* tag = NULL; + int32_t tagLen = 0; + if (streamStateGetParTag(pState, pBlock->info.id.groupId, &tag, &tagLen) == 0) { + pBlock->info.tagLen = tagLen; + void* pTag = taosMemoryRealloc(pBlock->info.pTag, tagLen); + if (pTag == NULL) { + tdbFree(tag); + taosMemoryFree(pBlock->info.pTag); + pBlock->info.pTag = NULL; + pBlock->info.tagLen = 0; + return; + } + pBlock->info.pTag = pTag; + memcpy(pBlock->info.pTag, tag, tagLen); + tdbFree(tag); + return; } else { - pBlock->info.parTbName[0] = 0; + pBlock->info.pTag = NULL; } + tdbFree(tag); +} +#endif - if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { - streamStatePutParName(pState, pBlock->info.id.groupId, pBlock->info.parTbName); +static void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { + SExprSupp* pTbNameCalSup = &pInfo->tbnameCalSup; + SStreamState* pState = pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState; + blockDataCleanup(pInfo->pCreateTbRes); + if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) { + pBlock->info.parTbName[0] = 0; + } else { + appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup, &pInfo->tagCalSup, + pBlock->info.id.groupId, pBlock, 0, pInfo->pCreateTbRes); } - - blockDataDestroy(pSrcBlock); - blockDataDestroy(pResBlock); } void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, @@ -1460,7 +1459,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock dumyInfo.cur.pageId = -1; bool isClosed = false; STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; - bool overDue = isOverdue(tsCol[rowId], &pInfo->twAggSup); + bool overDue = isOverdue(tsCol[rowId], &pInfo->twAggSup); if (pInfo->igExpired && overDue) { continue; } @@ -1565,25 +1564,29 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("queue scan called"); - if (pTaskInfo->streamInfo.pReq != NULL) { - if (pInfo->tqReader->pMsg == NULL) { - pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq; - const SSubmitReq* pSubmit = pInfo->tqReader->pMsg; - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { - qError("submit msg messed up when initing stream submit block %p", pSubmit); - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; - return NULL; + if (pTaskInfo->streamInfo.submit.msgStr != NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { + /*pInfo->tqReader->pMsg = pTaskInfo->streamInfo.pReq;*/ + + /*const SSubmitReq* pSubmit = pInfo->tqReader->pMsg;*/ + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + /*void* msgStr = pTaskInfo->streamInfo.*/ + SPackedData submit = pTaskInfo->streamInfo.submit; + if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + ASSERT(0); } } blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1596,15 +1599,17 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->pMsg = NULL; - pTaskInfo->streamInfo.pReq = NULL; + pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->setMsg = 0; + pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows", pResult->info.rows); + qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64, pResult->info.rows, + pResult->info.window.skey, pResult->info.window.ekey); pTaskInfo->streamInfo.returned = 1; return pResult; } else { @@ -1725,6 +1730,24 @@ static void setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) { } } +static void doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey) { + if (pInfo->pUpdateInfo) { + checkUpdateData(pInfo, true, pInfo->pRes, true); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey); + if (pInfo->pUpdateDataRes->info.rows > 0) { + pInfo->updateResIndex = 0; + if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { + pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + // return pInfo->pUpdateDataRes; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; + } + } + } +} + static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1765,17 +1788,32 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->blockRecoverContiCnt = 0; return NULL; } - SSDataBlock* pBlock = doTableScan(pInfo->pTableScanOp); - if (pBlock != NULL) { + + switch (pInfo->scanMode) { + case STREAM_SCAN_FROM_RES: { + pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; + } break; + default: + break; + } + + pInfo->pRecoverRes = doTableScan(pInfo->pTableScanOp); + if (pInfo->pRecoverRes != NULL) { pInfo->blockRecoverContiCnt++; - calBlockTbName(pInfo, pBlock); + calBlockTbName(pInfo, pInfo->pRecoverRes); if (pInfo->pUpdateInfo) { - TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pBlock, pInfo->primaryTsIndex); + TSKEY maxTs = updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); } - qDebug("stream recover scan get block, rows %d", pBlock->info.rows); - printDataBlock(pBlock, "scan recover"); - return pBlock; + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; + } + qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + printDataBlock(pInfo->pRecoverRes, "scan recover"); + return pInfo->pRecoverRes; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; @@ -1800,8 +1838,9 @@ FETCH_NEXT_BLOCK: } int32_t current = pInfo->validBlockIndex++; - SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); - if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) { + SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current); + SSDataBlock* pBlock = pPacked->pDataBlock; + if (pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } // TODO move into scan @@ -1876,8 +1915,11 @@ FETCH_NEXT_BLOCK: qDebug("scan mode %d", pInfo->scanMode); switch (pInfo->scanMode) { case STREAM_SCAN_FROM_RES: { - blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; + doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey); + doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + pInfo->pRes->info.dataLoad = 1; + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); return pInfo->pRes; } break; case STREAM_SCAN_FROM_DELETE_DATA: { @@ -1930,7 +1972,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->pMsg == NULL) { + if (pInfo->tqReader->msg2.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -1938,22 +1980,22 @@ FETCH_NEXT_BLOCK: return NULL; } - int32_t current = pInfo->validBlockIndex++; - SSubmitReq* pSubmit = taosArrayGetP(pInfo->pBlockLists, current); - if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) { + int32_t current = pInfo->validBlockIndex++; + SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); + /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ + if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); - pInfo->tqReader->pMsg = NULL; continue; } } blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextDataBlock2(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader); + int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; @@ -1968,22 +2010,12 @@ FETCH_NEXT_BLOCK: continue; } - if (pInfo->pUpdateInfo) { - checkUpdateData(pInfo, true, pInfo->pRes, true); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); - if (pInfo->pUpdateDataRes->info.rows > 0) { - pInfo->updateResIndex = 0; - if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { - pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { - pInfo->scanMode = STREAM_SCAN_FROM_RES; - return pInfo->pUpdateDataRes; - } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { - pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; - } - } + if (pInfo->pCreateTbRes->info.rows > 0) { + pInfo->scanMode = STREAM_SCAN_FROM_RES; + return pInfo->pCreateTbRes; } + doCheckUpdate(pInfo, pBlockInfo->window.ekey); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); @@ -1995,10 +2027,8 @@ FETCH_NEXT_BLOCK: if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) { break; } else { - pInfo->tqReader->pMsg = NULL; continue; } - /*blockDataCleanup(pInfo->pRes);*/ } // record the scan action. @@ -2046,6 +2076,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { if (pInfo->dataReader && tsdbNextDataBlock(pInfo->dataReader)) { if (isTaskKilled(pTaskInfo)) { + tsdbReleaseDataBlock(pInfo->dataReader); longjmp(pTaskInfo->env, pTaskInfo->code); } @@ -2204,6 +2235,7 @@ static void destroyStreamScanOperatorInfo(void* param) { } cleanupExprSupp(&pStreamScan->tbnameCalSup); + cleanupExprSupp(&pStreamScan->tagCalSup); updateInfoDestroy(pStreamScan->pUpdateInfo); blockDataDestroy(pStreamScan->pRes); @@ -2211,6 +2243,7 @@ static void destroyStreamScanOperatorInfo(void* param) { blockDataDestroy(pStreamScan->pPullDataRes); blockDataDestroy(pStreamScan->pDeleteDataRes); blockDataDestroy(pStreamScan->pUpdateDataRes); + blockDataDestroy(pStreamScan->pCreateTbRes); taosArrayDestroy(pStreamScan->pBlockLists); taosMemoryFree(pStreamScan); } @@ -2266,7 +2299,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pTableScanNode->pTags != NULL) { int32_t numOfTags; - SExprInfo* pTagExpr = createExprInfo(pTableScanNode->pTags, NULL, &numOfTags); + SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags); if (pTagExpr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2277,7 +2310,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } } - pInfo->pBlockLists = taosArrayInit(4, POINTER_BYTES); + pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData)); if (pInfo->pBlockLists == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _error; @@ -2298,13 +2331,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; pTaskInfo->streamInfo.lastStatus.uid = -1; -// code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, -// &pTSInfo->base.dataReader, NULL); -// if (code != 0) { -// terrno = code; -// destroyTableScanOperatorInfo(pTableScanOp); -// goto _error; -// } } if (pHandle->initTqReader) { @@ -2325,6 +2351,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; + pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); + blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); // set the extract column id to streamHandle tqReaderSetColIdList(pInfo->tqReader, pColIds); @@ -2544,8 +2572,10 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { } STsdbReader* reader = pInfo->base.dataReader; + qTrace("tsdb/read-table-data: %p, enter next reader", reader); while (tsdbNextDataBlock(reader)) { if (isTaskKilled(pTaskInfo)) { + tsdbReleaseDataBlock(reader); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -2578,6 +2608,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; + qTrace("tsdb/read-table-data: %p, close reader", reader); tsdbReaderClose(pInfo->base.dataReader); pInfo->base.dataReader = NULL; return pBlock; @@ -2737,7 +2768,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* } bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%d, limit:%"PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, pInfo->limitInfo.numOfOutputRows); if (limitReached) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index dc3ab79afba0eeb14bd43de2f8b3aa06164c24b7..d84bdddd374e5b3dc22ffccaeaab8259d56a2f7a 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -46,8 +46,9 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pOperator->pTaskInfo = pTaskInfo; SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); + int32_t numOfCols = 0; + pOperator->exprSupp.pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); + pOperator->exprSupp.numOfExprs = numOfCols; int32_t numOfOutputCols = 0; int32_t code = @@ -56,7 +57,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* goto _error; } - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + pOperator->exprSupp.pCtx = + createSqlFunctionCtx(pOperator->exprSupp.pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); initResultSizeInfo(&pOperator->resultInfo, 1024); code = filterInitFromNode((SNode*)pSortNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); if (code != TSDB_CODE_SUCCESS) { @@ -68,8 +70,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); setOperatorInfo(pOperator, "SortOperator", QUERY_NODE_PHYSICAL_PLAN_SORT, true, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; + // lazy evaluation for the following parameter since the input datablock is not known till now. // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 3a7c7bda245ba5c398c17e2ec47373a3c07a788c..24f42ff178c45660a13f58df2f3c025eb3e78048 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -133,12 +133,16 @@ static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); static void destroySysScanOperator(void* param); static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); -static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse); +static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse, bool* equal); static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, SMetaReader* smrChildTable, const char* dbname, const char* tableName, int32_t* pNumOfRows, const SSDataBlock* dataBlock); +static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, const char* dbname, int32_t* pNumOfRows, + const SSDataBlock* dataBlock, char* tName, SSchemaWrapper* schemaRow, + char* tableType); + static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, SFilterInfo* pFilterInfo); @@ -159,7 +163,8 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); @@ -177,9 +182,9 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false; + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); @@ -193,9 +198,9 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false, equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; SMetaFltParam param = {.suid = 0, @@ -203,6 +208,7 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { .type = TSDB_DATA_TYPE_VARCHAR, .val = pVal->datum.p, .reverse = reverse, + .equal = equal, .filterFunc = func}; return -1; } @@ -212,9 +218,9 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false, equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; SMetaFltParam param = {.suid = 0, @@ -222,6 +228,7 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .type = TSDB_DATA_TYPE_BIGINT, .val = &pVal->datum.i, .reverse = reverse, + .equal = equal, .filterFunc = func}; int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); @@ -234,8 +241,8 @@ int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -246,8 +253,8 @@ int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -258,8 +265,8 @@ int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -270,8 +277,8 @@ int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -282,8 +289,8 @@ int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -350,12 +357,17 @@ static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name, SExecTaskInfo* pTaskInfo); void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode); -static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, - SSDataBlock* pBlock); -__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { + +static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, + SSDataBlock* pBlock); + +__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse, bool* equal) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { *reverse = true; } + if (ctype == OP_TYPE_EQUAL) { + *equal = true; + } if (ctype == OP_TYPE_LOWER_THAN) return optSysFilterFuncImpl__LowerThan; else if (ctype == OP_TYPE_LOWER_EQUAL) @@ -412,6 +424,176 @@ static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { return false; } +static SSDataBlock* sysTableScanUserCols(SOperatorInfo* pOperator) { + qDebug("sysTableScanUserCols get cols start"); + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_COLS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->req.filterTb[0]) { + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->req.filterTb); + + SMetaReader smrTable = {0}; + metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); + int32_t code = metaGetTableEntryByName(&smrTable, pInfo->req.filterTb); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + if (smrTable.me.type == TSDB_SUPER_TABLE) { + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + if (smrTable.me.type == TSDB_CHILD_TABLE) { + int64_t suid = smrTable.me.ctbEntry.suid; + metaReaderClear(&smrTable); + metaReaderInit(&smrTable, pInfo->readHandle.meta, 0); + code = metaGetTableEntryByUid(&smrTable, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + metaReaderClear(&smrTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + } + + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + SSchemaWrapper* schemaRow = NULL; + if (smrTable.me.type == TSDB_SUPER_TABLE) { + schemaRow = &smrTable.me.stbEntry.schemaRow; + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + } else if (smrTable.me.type == TSDB_NORMAL_TABLE) { + schemaRow = &smrTable.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + metaReaderClear(&smrTable); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + + int32_t ret = 0; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + SHashObj* stableSchema = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + taosHashSetFreeFp(stableSchema, tDeleteSSchemaWrapperForHash); + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_TABLE_MAX)) == 0) { + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + SSchemaWrapper* schemaRow = NULL; + + if (pInfo->pCur->mr.me.type == TSDB_SUPER_TABLE) { + qDebug("sysTableScanUserCols cursor get super table"); + void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t)); + if (schema == NULL) { + SSchemaWrapper* schemaWrapper = tCloneSSchemaWrapper(&pInfo->pCur->mr.me.stbEntry.schemaRow); + taosHashPut(stableSchema, &pInfo->pCur->mr.me.uid, sizeof(int64_t), &schemaWrapper, POINTER_BYTES); + } + continue; + } else if (pInfo->pCur->mr.me.type == TSDB_CHILD_TABLE) { + qDebug("sysTableScanUserCols cursor get child table"); + STR_TO_VARSTR(typeName, "CHILD_TABLE"); + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + int64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + void* schema = taosHashGet(stableSchema, &pInfo->pCur->mr.me.ctbEntry.suid, sizeof(int64_t)); + if (schema != NULL) { + schemaRow = *(SSchemaWrapper**)schema; + } else { + tDecoderClear(&pInfo->pCur->mr.coder); + int code = metaGetTableEntryByUid(&pInfo->pCur->mr, suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + qError("sysTableScanUserCols get meta by suid:%" PRId64 " error, code:%d", suid, code); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + taosHashCleanup(stableSchema); + return NULL; + } + schemaRow = &pInfo->pCur->mr.me.stbEntry.schemaRow; + } + } else if (pInfo->pCur->mr.me.type == TSDB_NORMAL_TABLE) { + qDebug("sysTableScanUserCols cursor get normal table"); + schemaRow = &pInfo->pCur->mr.me.ntbEntry.schemaRow; + STR_TO_VARSTR(typeName, "NORMAL_TABLE"); + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + } else { + qDebug("sysTableScanUserCols cursor get invalid table"); + continue; + } + + sysTableUserColsFillOneTableCols(pInfo, dbname, &numOfRows, dataBlock, tableName, schemaRow, typeName); + + if (numOfRows >= pOperator->resultInfo.capacity) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + taosHashCleanup(stableSchema); + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + blockDataDestroy(dataBlock); + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + qDebug("sysTableScanUserCols get cols success, rows:%" PRIu64, pInfo->loadInfo.totalRows); + + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; @@ -491,7 +673,7 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { } bool blockFull = false; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { continue; } @@ -736,6 +918,66 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, return TSDB_CODE_SUCCESS; } +static int32_t sysTableUserColsFillOneTableCols(const SSysTableScanInfo* pInfo, const char* dbname, int32_t* pNumOfRows, + const SSDataBlock* dataBlock, char* tName, SSchemaWrapper* schemaRow, + char* tableType) { + if (schemaRow == NULL) { + qError("sysTableUserColsFillOneTableCols schemaRow is NULL"); + return TSDB_CODE_SUCCESS; + } + int32_t numOfRows = *pNumOfRows; + + int32_t numOfCols = schemaRow->nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tName, false); + + // database name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, tableType, false); + + // col name + char colName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(colName, schemaRow->pSchema[i].name); + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, colName, false); + + // col type + int8_t colType = schemaRow->pSchema[i].type; + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + char colTypeStr[VARSTR_HEADER_SIZE + 32]; + int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); + if (colType == TSDB_DATA_TYPE_VARCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)(schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE)); + } else if (colType == TSDB_DATA_TYPE_NCHAR) { + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((schemaRow->pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(colTypeStr, colTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (const char*)&schemaRow->pSchema[i].bytes, false); + + for (int32_t j = 6; j <= 8; ++j) { + pColInfoData = taosArrayGet(dataBlock->pDataBlock, j); + colDataSetNULL(pColInfoData, numOfRows); + } + ++numOfRows; + } + + *pNumOfRows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { size_t size = 0; const SSysTableMeta* pMeta = NULL; @@ -1037,7 +1279,7 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + while ((ret = metaTbCursorNext(pInfo->pCur, TSDB_SUPER_TABLE)) == 0) { STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); // table name @@ -1323,6 +1565,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pInfo->showRewrite) { getDBNameFromCondition(pInfo->pCondition, dbName); sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0) { + getDBNameFromCondition(pInfo->pCondition, dbName); + if (dbName[0]) sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + sysTableIsCondOnOneTable(pInfo->pCondition, pInfo->req.filterTb); } SSDataBlock* pBlock = NULL; @@ -1330,6 +1576,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { pBlock = sysTableScanUserTables(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { pBlock = sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->readHandle.mnd == NULL) { + pBlock = sysTableScanUserCols(pOperator); } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && IS_SYS_DBNAME(dbName)) { pBlock = sysTableScanUserSTables(pOperator); @@ -1359,8 +1607,7 @@ static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScan if (pInfo->tbnameSlotId != -1) { SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId); char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0}; - memcpy(varDataVal(varTbName), name, strlen(name)); - varDataSetLen(varTbName, strlen(name)); + STR_TO_VARSTR(varTbName, name); colDataSetNItems(pColumnInfoData, 0, varTbName, pBlock->info.rows); } @@ -1407,7 +1654,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca tsem_wait(&pInfo->ready); if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + qError("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); return NULL; } @@ -1443,6 +1690,7 @@ static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableSca SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo) { + int32_t code = TDB_CODE_SUCCESS; SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1453,7 +1701,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t num = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1461,7 +1709,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan extractTbnameSlotId(pInfo, pScanNode); pInfo->accountId = pScanPhyNode->accountId; - pInfo->pUser = taosMemoryStrDup((void*)pUser); + pInfo->pUser = taosStrdup((void*)pUser); pInfo->sysInfo = pScanPhyNode->sysInfo; pInfo->showRewrite = pScanPhyNode->showRewrite; pInfo->pRes = createDataBlockFromDescNode(pDescNode); @@ -1530,7 +1778,8 @@ void destroySysScanOperator(void* param) { const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_COLS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -1929,7 +2178,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { // make the valgrind happy that all memory buffer has been initialized already. if (slotId != 0) { SColumnInfoData* p1 = taosArrayGet(pBlock->pDataBlock, 0); - int64_t v = 0; + int64_t v = 0; colDataSetInt64(p1, 0, &v); } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index a28def17de013cd1472081916a245ed078fbc0f6..911700be8534d7b6b7762854b82bd7ceaad12489 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -91,7 +91,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i); - if (!IS_MATHABLE_TYPE(pColInfoData->info.type)) { continue; } @@ -100,7 +99,6 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo // TODO: optimize to ignore null values for linear interpolation. if (!pLinearInfo->isStartSet) { if (!colDataIsNull_s(pColInfoData, rowIndex)) { - pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex); char* p = colDataGetData(pColInfoData, rowIndex); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -144,10 +142,8 @@ static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlo } } } - } - static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* pSliceInfo, SSDataBlock* pBlock) { if (pBlock->info.rows < pBlock->info.capacity) { return TSDB_CODE_SUCCESS; @@ -160,8 +156,8 @@ static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* return TSDB_CODE_SUCCESS; } - -static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { +static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, + bool beforeTs) { int32_t rows = pResBlock->info.rows; timeSliceEnsureBlockCapacity(pSliceInfo, pResBlock); // todo set the correct primary timestamp column @@ -177,6 +173,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataSetVal(pDst, rows, (char*)&pSliceInfo->current, false); continue; + } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + bool isFilled = true; + colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); + continue; } int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; @@ -290,6 +290,9 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); + } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + bool isFilled = false; + colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); } else { int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot); @@ -308,7 +311,6 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* return; } - static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { if (pInfo->pPrevRow != NULL) { return TSDB_CODE_SUCCESS; @@ -480,11 +482,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && + pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { - pSliceInfo->current = - taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, + pInterval->precision); } } @@ -504,7 +507,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepLinearInfo(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) { + if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && + pSliceInfo->fillType == TSDB_FILL_LINEAR) { break; } else { pSliceInfo->current = @@ -599,14 +603,15 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); + pOperator->fpSet = + createOperatorFpSet(optrDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, optrDefaultBufFn, NULL); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 62d68d5ca25a7c3812872be8008766e91d40020d..20d4f46eaf5e2f87dfe91515aa7885e434cb74ad 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -842,70 +842,61 @@ static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) { return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); } -static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SHashObj* pUpdatedMap) { - SResKeyPos* newPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - if (newPos == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - newPos->groupId = groupId; - newPos->pos = (SResultRowPosition){.pageId = pageId, .offset = offset}; - *(int64_t*)newPos->key = ts; +static int32_t saveWinResult(int64_t ts, uint64_t groupId, SSHashObj* pUpdatedMap) { SWinKey key = {.ts = ts, .groupId = groupId}; - if (taosHashPut(pUpdatedMap, &key, sizeof(SWinKey), &newPos, sizeof(void*)) != TSDB_CODE_SUCCESS) { - taosMemoryFree(newPos); - } - + tSimpleHashPut(pUpdatedMap, &key, sizeof(SWinKey), NULL, 0); return TSDB_CODE_SUCCESS; } -static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) { - return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); +static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SSHashObj* pUpdatedMap) { + return saveWinResult(ts, groupId, pUpdatedMap); } -static void removeResults(SArray* pWins, SHashObj* pUpdatedMap) { +static void removeResults(SArray* pWins, SSHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { SWinKey* pW = taosArrayGet(pWins, i); - void* tmp = taosHashGet(pUpdatedMap, pW, sizeof(SWinKey)); + void* tmp = tSimpleHashGet(pUpdatedMap, pW, sizeof(SWinKey)); if (tmp) { void* value = *(void**)tmp; taosMemoryFree(value); - taosHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); + tSimpleHashRemove(pUpdatedMap, pW, sizeof(SWinKey)); } } } -int32_t compareWinRes(void* pKey, void* data, int32_t index) { - SArray* res = (SArray*)data; - SWinKey* pDataPos = taosArrayGet(res, index); - SResKeyPos* pRKey = (SResKeyPos*)pKey; - if (pRKey->groupId > pDataPos->groupId) { +int32_t compareWinKey(void* pKey, void* data, int32_t index) { + SArray* res = (SArray*)data; + SWinKey* pDataPos = taosArrayGet(res, index); + SWinKey* pWKey = (SWinKey*)pKey; + + if (pWKey->groupId > pDataPos->groupId) { return 1; - } else if (pRKey->groupId < pDataPos->groupId) { + } else if (pWKey->groupId < pDataPos->groupId) { return -1; } - if (*(int64_t*)pRKey->key > pDataPos->ts) { + if (pWKey->ts > pDataPos->ts) { return 1; - } else if (*(int64_t*)pRKey->key < pDataPos->ts) { + } else if (pWKey->ts < pDataPos->ts) { return -1; } return 0; } -static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { +static void removeDeleteResults(SSHashObj* pUpdatedMap, SArray* pDelWins) { taosArraySort(pDelWins, winKeyCmprImpl); taosArrayRemoveDuplicate(pDelWins, winKeyCmprImpl, NULL); int32_t delSize = taosArrayGetSize(pDelWins); - if (taosHashGetSize(pUpdatedMap) == 0 || delSize == 0) { + if (tSimpleHashGetSize(pUpdatedMap) == 0 || delSize == 0) { return; } - void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - SResKeyPos* pResKey = *(SResKeyPos**)pIte; - int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes); - if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pUpdatedMap, pIte, &iter)) != NULL) { + SWinKey* pResKey = tSimpleHashGetKey(pIte, NULL); + int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinKey); + if (index >= 0 && 0 == compareWinKey(pResKey, pDelWins, index)) { taosArrayRemove(pDelWins, index); delSize = taosArrayGetSize(pDelWins); } @@ -1354,7 +1345,7 @@ static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId) } static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDataBlock* pBlock, SArray* pUpWins, - SHashObj* pUpdatedMap) { + SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; @@ -1390,28 +1381,21 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, SSDa taosArrayPush(pUpWins, &winRes); } if (pUpdatedMap) { - void* tmp = taosHashGet(pUpdatedMap, &winRes, sizeof(SWinKey)); - if (tmp) { - void* value = *(void**)tmp; - taosMemoryFree(value); - taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); - } + tSimpleHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); } while (win.ekey <= endTsCols[i]); } } -static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { +static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) { void* pIte = NULL; - size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*)key; - TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); - SResultRowPosition* pPos = (SResultRowPosition*)pIte; - int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins); + SWinKey* pKey = tSimpleHashGetKey(pIte, NULL); + uint64_t groupId = pKey->groupId; + TSKEY ts = pKey->ts; + int32_t code = saveWinResult(ts, groupId, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1419,36 +1403,16 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { return TSDB_CODE_SUCCESS; } -int32_t compareWinKey(void* pKey, void* data, int32_t index) { - SArray* res = (SArray*)data; - SWinKey* pDataPos = taosArrayGet(res, index); - SWinKey* pWKey = (SWinKey*)pKey; - - if (pWKey->groupId > pDataPos->groupId) { - return 1; - } else if (pWKey->groupId < pDataPos->groupId) { - return -1; - } - - if (pWKey->ts > pDataPos->ts) { - return 1; - } else if (pWKey->ts < pDataPos->ts) { - return -1; - } - return 0; -} - static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, - SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pDelWins, + SHashObj* pPullDataMap, SSHashObj* closeWins, SArray* pDelWins, SOperatorInfo* pOperator) { qDebug("===stream===close interval window"); void* pIte = NULL; - size_t keyLen = 0; int32_t iter = 0; SStreamIntervalOperatorInfo* pInfo = pOperator->info; int32_t delSize = taosArrayGetSize(pDelWins); while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); + void* key = tSimpleHashGetKey(pIte, NULL); SWinKey* pWinKey = (SWinKey*)key; if (delSize > 0) { int32_t index = binarySearchCom(pDelWins, delSize, pWinKey, TSDB_ORDER_DESC, compareWinKey); @@ -1650,7 +1614,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { } nodesDestroyNode((SNode*)pInfo->pPhyNode); colDataDestroy(&pInfo->twAggSup.timeWindowData); - cleanupGroupResInfo(&pInfo->groupResInfo); + pInfo->groupResInfo.pRows = taosArrayDestroy(pInfo->groupResInfo.pRows); cleanupExprSupp(&pInfo->scalarSupp); taosMemoryFreeClear(param); @@ -1742,7 +1706,7 @@ void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { } SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, - SExecTaskInfo* pTaskInfo, bool isStream) { + SExecTaskInfo* pTaskInfo) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1975,6 +1939,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes; } +// todo make this as an non-blocking operator SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); @@ -2158,7 +2123,7 @@ bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0); } -static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SHashObj* pUpdatedMap) { +static void rebuildIntervalWindow(SOperatorInfo* pOperator, SArray* pWinArray, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t size = taosArrayGetSize(pWinArray); @@ -2344,7 +2309,8 @@ static void clearFunctionContext(SExprSupp* pSup) { } } -void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { +void doBuildStreamIntervalResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, + SGroupResInfo* pGroupResInfo) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // set output datablock version pBlock->info.version = pTaskInfo->version; @@ -2371,7 +2337,7 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN } static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, - SHashObj* pUpdatedMap) { + SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); @@ -2507,12 +2473,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - TSKEY maxTs = INT64_MIN; - TSKEY minTs = INT64_MAX; - - SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + SExprSupp* pSup = &pOperator->exprSupp; qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); @@ -2533,7 +2495,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -2560,7 +2522,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -2568,10 +2530,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } } - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, sizeof(SWinKey)); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = tSimpleHashInit(1024, hashFn); + } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2589,35 +2555,39 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, delWins, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; doDeleteWindows(pChildOp, &pChildInfo->interval, pBlock, NULL, NULL); - rebuildIntervalWindow(pOperator, delWins, pUpdatedMap); + rebuildIntervalWindow(pOperator, delWins, pInfo->pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); continue; } - removeResults(delWins, pUpdatedMap); + removeResults(delWins, pInfo->pUpdatedMap); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pUpdatedMap); - if (taosArrayGetSize(pUpdated) > 0) { + doDeleteWindows(pOperator, &pInfo->interval, pBlock, NULL, pInfo->pUpdatedMap); + if (taosArrayGetSize(pInfo->pUpdated) > 0) { break; } continue; } else if (pBlock->info.type == STREAM_PULL_OVER && IS_FINAL_OP(pInfo)) { processPullOver(pBlock, pInfo->pPullDataMap, &pInfo->interval); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -2625,7 +2595,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -2645,29 +2615,30 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.id.groupId, NULL); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); } - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); if (IS_FINAL_OP(pInfo)) { closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, - pInfo->pPullDataMap, pUpdatedMap, pInfo->pDelWins, pOperator); + pInfo->pPullDataMap, pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs); } pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; - void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { + taosArrayPush(pInfo->pUpdated, pIte); } - taosHashCleanup(pUpdatedMap); - taosArraySort(pUpdated, resultrowComparAsc); + tSimpleHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; + taosArraySort(pInfo->pUpdated, winKeyCmprImpl); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); @@ -2684,7 +2655,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -2805,6 +2776,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -3248,10 +3221,9 @@ static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) static int32_t copyUpdateResult(SSHashObj* pStUpdated, SArray* pUpdated) { void* pIte = NULL; - size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pStUpdated, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); + void* key = tSimpleHashGetKey(pIte, NULL); taosArrayPush(pUpdated, key); } taosArraySort(pUpdated, sessionKeyCompareAsc); @@ -3265,13 +3237,12 @@ void doBuildDeleteDataBlock(SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlo return; } blockDataEnsureCapacity(pBlock, size); - size_t keyLen = 0; int32_t iter = 0; while (((*Ite) = tSimpleHashIterate(pStDeleted, *Ite, &iter)) != NULL) { if (pBlock->info.rows + 1 > pBlock->info.capacity) { break; } - SSessionKey* res = tSimpleHashGetKey(*Ite, &keyLen); + SSessionKey* res = tSimpleHashGetKey(*Ite, NULL); SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)&res->win.skey, false); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -3360,7 +3331,6 @@ static void rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, SS int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed) { void* pIte = NULL; - size_t keyLen = 0; int32_t iter = 0; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { SResultWindowInfo* pWinInfo = pIte; @@ -3371,7 +3341,7 @@ int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHa return code; } } - SSessionKey* pKey = tSimpleHashGetKey(pIte, &keyLen); + SSessionKey* pKey = tSimpleHashGetKey(pIte, NULL); tSimpleHashIterateRemove(pHashMap, pKey, sizeof(SSessionKey), &pIte, &iter); } } @@ -3437,7 +3407,6 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - TSKEY maxTs = INT64_MIN; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -3457,10 +3426,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); // SResKeyPos + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3473,21 +3446,25 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); // gap must be 0 doDeleteTimeWindows(pAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, NULL); - rebuildSessionWindow(pOperator, pWins, pStUpdated); + rebuildSessionWindow(pOperator, pWins, pInfo->pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pAggSup->pResultRows, pStUpdated); + getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3496,7 +3473,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, pInfo->pStDeleted, IS_FINAL_OP(pInfo)); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3513,20 +3490,20 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { setInputDataBlock(&pChildOp->exprSupp, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.watermark); } - - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pStUpdated); + closeSessionWindow(pAggSup->pResultRows, &pInfo->twAggSup, pInfo->pStUpdated); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -3611,6 +3588,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pStUpdated = NULL; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3671,10 +3650,14 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pStUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pStUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pStUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -3689,13 +3672,17 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { // gap must be 0 SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pStUpdated, pWins); + removeSessionResults(pInfo->pStUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pStDeleted); taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pStUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pStUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -3704,18 +3691,20 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false); + doStreamSessionAggImpl(pOperator, pBlock, pInfo->pStUpdated, NULL, false); maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - copyUpdateResult(pStUpdated, pUpdated); - removeSessionResults(pInfo->pStDeleted, pUpdated); - tSimpleHashCleanup(pStUpdated); + copyUpdateResult(pInfo->pStUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pStDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pStUpdated); + pInfo->pStUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); #if 0 @@ -3975,7 +3964,6 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -3993,10 +3981,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SSHashObj* pSeUpdated = tSimpleHashInit(64, hashFn); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SSessionKey)); + } + if (!pInfo->pSeUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4008,13 +4000,17 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(pSeUpdated, pWins); + removeSessionResults(pInfo->pSeUpdated, pWins); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllSessionWindow(pInfo->streamAggSup.pResultRows, pSeUpdated); + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pInfo->scalarSupp.pExprInfo != NULL) { @@ -4023,19 +4019,20 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); - maxTs = TMAX(maxTs, pBlock->info.window.ekey); + doStreamStateAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pSeUpdated); - copyUpdateResult(pSeUpdated, pUpdated); - removeSessionResults(pInfo->pSeDeleted, pUpdated); - tSimpleHashCleanup(pSeUpdated); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + copyUpdateResult(pInfo->pSeUpdated, pInfo->pUpdated); + removeSessionResults(pInfo->pSeDeleted, pInfo->pUpdated); + tSimpleHashCleanup(pInfo->pSeUpdated); + pInfo->pSeUpdated = NULL; - initGroupResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); #if 0 @@ -4116,6 +4113,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->pChildren = NULL; pInfo->ignoreExpiredData = pStateNode->window.igExpired; + pInfo->pUpdated = NULL; + pInfo->pSeUpdated = NULL; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4729,8 +4728,6 @@ _error: static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - int64_t maxTs = INT64_MIN; - int64_t minTs = INT64_MAX; SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_EXEC_DONE) { @@ -4744,7 +4741,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; @@ -4758,10 +4755,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(4, sizeof(SWinKey)); + } + if (!pInfo->pUpdatedMap) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pUpdatedMap = tSimpleHashInit(1024, hashFn); + } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); @@ -4775,11 +4775,16 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + printDataBlock(pBlock, "single interval"); + return pBlock; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); } if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { @@ -4800,28 +4805,29 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - minTs = TMIN(minTs, pBlock->info.window.skey); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, pBlock->info.window.skey); - doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.id.groupId, pInfo->pUpdatedMap); } - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); - pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); pOperator->status = OP_RES_TO_RETURN; - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, - pInfo->pDelWins, pOperator); + removeDeleteResults(pInfo->pUpdatedMap, pInfo->pDelWins); + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, + pInfo->pUpdatedMap, pInfo->pDelWins, pOperator); - void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pInfo->pUpdatedMap, pIte, &iter)) != NULL) { + SWinKey* pKey = tSimpleHashGetKey(pIte, NULL); + taosArrayPush(pInfo->pUpdated, pKey); } - taosArraySort(pUpdated, resultrowComparAsc); - - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + taosArraySort(pInfo->pUpdated, winKeyCmprImpl); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - taosHashCleanup(pUpdatedMap); + tSimpleHashCleanup(pInfo->pUpdatedMap); + pInfo->pUpdatedMap = NULL; #if 0 char* pBuf = streamStateIntervalDump(pInfo->pState); @@ -4835,7 +4841,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildStreamIntervalResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; @@ -4928,6 +4934,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->delKey.ts = INT64_MAX; pInfo->delKey.groupId = 0; pInfo->numOfDatapack = 0; + pInfo->pUpdated = NULL; + pInfo->pUpdatedMap = NULL; setOperatorInfo(pOperator, "StreamIntervalOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4948,4 +4956,3 @@ _error: pTaskInfo->code = code; return NULL; } - diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 14de05cf6ecf1cdde61dc0f30367a5a53bb45156..6d734901ab3e43a165bdba05690abd222ff96385 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -90,7 +90,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page tsortSetComparFp(pSortHandle, msortComparFn); if (idstr != NULL) { - pSortHandle->idStr = strdup(idstr); + pSortHandle->idStr = taosStrdup(idstr); } return pSortHandle; @@ -212,6 +212,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { int32_t pageId = -1; void* pPage = getNewBufPage(pHandle->pBuf, &pageId); if (pPage == NULL) { + taosArrayDestroy(pPageIdList); blockDataDestroy(p); taosArrayDestroy(pPageIdList); return terrno; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index faf7a29dd0aa34031325dc9c0b49cf46e403e78a..0257b3d5e6c2e0062b00b79dab221f4ddb606d4a 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -466,7 +466,7 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -480,14 +480,21 @@ static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t le return code; } - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; + pFunc->node.resType = (SDataType){.bytes =tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateIsFilledPseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + // pseudo column do not need to check parameters + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes, .type = TSDB_DATA_TYPE_BOOL}; return TSDB_CODE_SUCCESS; } @@ -497,27 +504,45 @@ static int32_t translateTimezone(SFunctionNode* pFunc, char* pErrBuf, int32_t le } static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (2 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams < 2 || numOfParams > 11) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - // param1 - SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); - if (pValue->datum.i < 0 || pValue->datum.i > 100) { - return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(para1Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pValue->notReserved = true; - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; - if (!IS_NUMERIC_TYPE(para1Type) || (!IS_SIGNED_NUMERIC_TYPE(para2Type) && !IS_UNSIGNED_NUMERIC_TYPE(para2Type))) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + for (int32_t i = 1; i < numOfParams; ++i) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i); + pValue->notReserved = true; + + uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, i))->resType.type; + if (!IS_NUMERIC_TYPE(paraType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + double v = 0; + if (IS_INTEGER_TYPE(paraType)) { + v = (double)pValue->datum.i; + } else { + v = pValue->datum.d; + } + + if (v < 0 || v > 100) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } } // set result type - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + if (numOfParams > 2) { + pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_VARCHAR}; + } else { + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + } return TSDB_CODE_SUCCESS; } @@ -2266,8 +2291,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_REPEAT_SCAN_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translatePercentile, + .dataRequiredFunc = statisDataRequired, .getEnvFunc = getPercentileFuncEnv, .initFunc = percentileFunctionSetup, .processFunc = percentileFunction, @@ -3257,6 +3283,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = NULL, .finalizeFunc = NULL }, + { + .name = "_isfilled", + .type = FUNCTION_TYPE_ISFILLED, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC, + .translateFunc = translateIsFilledPseudoColumn, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = NULL, + .finalizeFunc = NULL + }, { .name = "_tags", .type = FUNCTION_TYPE_TAGS, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 958f1db120ce459643d9d7131c2ca7ae4cf17606..9986af1691ca4ddb2a3d58fad7d28013decae66a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -888,13 +888,6 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu return TSDB_CODE_SUCCESS; } -void releaseSource(STuplePos* pPos) { - if (pPos->pageId == -1) { - return; - } - // Todo(liuyao) relase row -} - // This function append the selectivity to subsidiaries function context directly, without fetching data // from intermediate disk based buf page void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) { @@ -926,10 +919,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) } } -void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { - releaseSource(pDestPos); - *pDestPos = *pSourcePos; -} +void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { *pDestPos = *pSourcePos; } int32_t minMaxCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t isMinFunc) { SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx); @@ -1606,7 +1596,7 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { int32_t type = pCol->info.type; SPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - if (pCtx->scanFlag == REPEAT_SCAN && pInfo->stage == 0) { + if (pCtx->scanFlag == MAIN_SCAN && pInfo->stage == 0) { pInfo->stage += 1; // all data are null, set it completed @@ -1689,26 +1679,63 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { } int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SVariant* pVal = &pCtx->param[1].param; - int32_t code = 0; - double v = 0; - - GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo*)GET_ROWCELL_INTERBUF(pResInfo); + int32_t code = 0; + double v = 0; + tMemBucket* pMemBucket = ppInfo->pMemBucket; if (pMemBucket != NULL && pMemBucket->total > 0) { // check for null - code = getPercentile(pMemBucket, v, &ppInfo->result); - } + if (pCtx->numOfParams > 2) { + char buf[512] = {0}; + size_t len = 1; - tMemBucketDestroy(pMemBucket); - if (code != TSDB_CODE_SUCCESS) { - return code; + varDataVal(buf)[0] = '['; + for (int32_t i = 1; i < pCtx->numOfParams; ++i) { + SVariant* pVal = &pCtx->param[i].param; + + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); + + int32_t code = getPercentile(pMemBucket, v, &ppInfo->result); + if (code != TSDB_CODE_SUCCESS) { + goto _fin_error; + } + + if (i == pCtx->numOfParams - 1) { + len += snprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf]", ppInfo->result); + } else { + len += snprintf(varDataVal(buf) + len, sizeof(buf) - VARSTR_HEADER_SIZE - len, "%.6lf, ", ppInfo->result); + } + } + + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + varDataSetLen(buf, len); + colDataAppend(pCol, pBlock->info.rows, buf, false); + + tMemBucketDestroy(pMemBucket); + return pResInfo->numOfRes; + } else { + SVariant* pVal = &pCtx->param[1].param; + + GET_TYPED_DATA(v, double, pVal->nType, &pVal->i); + + code = getPercentile(pMemBucket, v, &ppInfo->result); + if (code != TSDB_CODE_SUCCESS) { + goto _fin_error; + } + + tMemBucketDestroy(pMemBucket); + return functionFinalize(pCtx, pBlock); + } } - return functionFinalize(pCtx, pBlock); +_fin_error: + + tMemBucketDestroy(pMemBucket); + return code; } bool getApercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -5413,6 +5440,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) { } pDistInfo->numOfVgroups += (p1.numOfTables != 0 ? 1 : 0); + pDistInfo->numOfVgroups += (p1.numOfTables != 0 ? 1 : 0); for (int32_t i = 0; i < tListLen(pDistInfo->blockRowsHisto); ++i) { pDistInfo->blockRowsHisto[i] += p1.blockRowsHisto[i]; } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 1734b535f13dfe33f391187efe8ec59e86b94992..de381fadbd6efa25494559c5a9f63ab6a0422cf9 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -33,7 +33,14 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) (SFilePage *)taosMemoryCalloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage)); int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times); - SArray *pIdList = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + + SArray *pIdList; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + pIdList = *(SArray **)p; + } else { + return NULL; + } int32_t offset = 0; for (int32_t i = 0; i < taosArrayGetSize(pIdList); ++i) { @@ -512,8 +519,17 @@ int32_t getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction resetSlotInfo(pMemBucket); int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1); - SArray* list = *(SArray **)taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); - ASSERT(list != NULL && list->size > 0); + + SArray* list; + void *p = taosHashGet(pMemBucket->groupPagesMap, &groupId, sizeof(groupId)); + if (p != NULL) { + list = *(SArray **)p; + if (list == NULL || list->size <= 0) { + return -1; + } + } else { + return -1; + } for (int32_t f = 0; f < list->size; ++f) { int32_t *pageId = taosArrayGet(list, f); diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index a99c87b7f9a5bef05c8e641ab4aa826e7c5e7dfc..3ed66956e841b43d0744cbd2c7aa0ad6614599df 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -126,7 +126,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); idx->version = 1; - idx->path = tstrdup(path); + idx->path = taosStrdup(path); taosThreadMutexInit(&idx->mtx, NULL); tsem_init(&idx->sem, 0, 0); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 0bb454571adebae9695ae07cf7ff6af4efd9e95f..8b0e7125530fd557648d97106fc106132972c71b 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -22,8 +22,8 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 8 * 512 * 1024 // 8M -#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 +#define MEM_THRESHOLD 128 * 1024 * 1024 // 8M +#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 5 #define MEM_ESTIMATE_RADIO 1.5 static void idxMemRef(MemTable* tbl); @@ -340,7 +340,7 @@ IndexCache* idxCacheCreate(SIndex* idx, uint64_t suid, const char* colName, int8 cache->mem = idxInternalCacheCreate(type); cache->mem->pCache = cache; - cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? tstrdup(JSON_COLUMN) : tstrdup(colName); + cache->colName = IDX_TYPE_CONTAIN_EXTERN_TYPE(type, TSDB_DATA_TYPE_JSON) ? taosStrdup(JSON_COLUMN) : taosStrdup(colName); cache->type = type; cache->index = idx; cache->version = 0; @@ -767,7 +767,7 @@ static bool idxCacheIteratorNext(Iterate* itera) { iv->type = ct->operaType; iv->ver = ct->version; - iv->colVal = tstrdup(ct->colVal); + iv->colVal = taosStrdup(ct->colVal); taosArrayPush(iv->val, &ct->uid); } return next; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 5c14424b7ecb56a45405cd48daa018a5f8837da9..0d13db2f358587c349ef79b7411e8ac45810b883 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -383,12 +383,13 @@ static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) { //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } -static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse) { +static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse, bool *equal) { if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) { *reverse = true; } else { *reverse = false; } + if (type == QUERY_LESS_EQUAL) return sifLessEqual; else if (type == QUERY_LESS_THAN) @@ -398,6 +399,7 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve else if (type == QUERY_GREATER_THAN) return sifGreaterThan; else if (type == QUERY_TERM) { + *equal = true; return sifEqual; } return NULL; @@ -474,14 +476,15 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); indexMultiTermQueryDestroy(mtm); } else { - bool reverse; - FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); + bool reverse = false, equal = false; + FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse, &equal); SMetaFltParam param = {.suid = arg->suid, .cid = left->colId, .type = left->colValType, .val = right->condValue, .reverse = reverse, + .equal = equal, .filterFunc = filterFunc}; char buf[128] = {0}; diff --git a/source/libs/index/src/indexFstAutomation.c b/source/libs/index/src/indexFstAutomation.c index 385e832763228494af2b1562f915da14b6500a25..0c96d1aa0a9a62c351b1b7419b387a763daddfbc 100644 --- a/source/libs/index/src/indexFstAutomation.c +++ b/source/libs/index/src/indexFstAutomation.c @@ -164,7 +164,7 @@ FAutoCtx* automCtxCreate(void* data, AutomationType atype) { // add more search type } - ctx->data = (data != NULL ? strdup((char*)data) : NULL); + ctx->data = (data != NULL ? taosStrdup((char*)data) : NULL); ctx->type = atype; ctx->stdata = (void*)sv; return ctx; diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c index e148f211f23a6acbf9bd81eed2dff13761a7800a..8b513bfb2be3245c97fc9041d14f556c86145709 100644 --- a/source/libs/index/src/indexFstRegex.c +++ b/source/libs/index/src/indexFstRegex.c @@ -23,7 +23,7 @@ FstRegex *regexCreate(const char *str) { return NULL; } - regex->orig = tstrdup(str); + regex->orig = taosStrdup(str); // construct insts based on str SArray *insts = taosArrayInit(256, sizeof(uint8_t)); diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index d921ca7103c55d51421a10818b2b23964854869c..cdd1cc738693bf6dcc9c87b99b153370838a44be 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -804,7 +804,7 @@ TFileValue* tfileValueCreate(char* val) { if (tf == NULL) { return NULL; } - tf->colVal = tstrdup(val); + tf->colVal = taosStrdup(val); tf->tableId = taosArrayInit(32, sizeof(uint64_t)); return tf; } diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 4e9a85330299fb3689969335eb96b613b3a40c36..b889a2209a76fe1ebff7f73cd81fa4b841c63fd9 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -599,7 +599,7 @@ void validateTFile(char* arg) { std::thread threads[NUM_OF_THREAD]; // std::vector threads; SIndex* index = (SIndex*)taosMemoryCalloc(1, sizeof(SIndex)); - index->path = strdup(arg); + index->path = taosStrdup(arg); TFileReader* reader = tfileReaderOpen(index, 0, 20000000, "tag1"); for (int i = 0; i < NUM_OF_THREAD; i++) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 2b1819d5c0d8cd27dc70e0adfe8e44192835608d..b4f7ea866a21afa13d1f097709ae1ef27073659a 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -40,7 +40,7 @@ if (NULL == (pSrc)->fldname) { \ break; \ } \ - (pDst)->fldname = strdup((pSrc)->fldname); \ + (pDst)->fldname = taosStrdup((pSrc)->fldname); \ if (NULL == (pDst)->fldname) { \ return TSDB_CODE_OUT_OF_MEMORY; \ } \ @@ -295,6 +295,13 @@ static int32_t stateWindowNodeCopy(const SStateWindowNode* pSrc, SStateWindowNod return TSDB_CODE_SUCCESS; } +static int32_t eventWindowNodeCopy(const SEventWindowNode* pSrc, SEventWindowNode* pDst) { + CLONE_NODE_FIELD(pCol); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); + return TSDB_CODE_SUCCESS; +} + static int32_t sessionWindowNodeCopy(const SSessionWindowNode* pSrc, SSessionWindowNode* pDst) { CLONE_NODE_FIELD_EX(pCol, SColumnNode*); CLONE_NODE_FIELD_EX(pGap, SValueNode*); @@ -464,6 +471,8 @@ static int32_t logicWindowCopy(const SWindowLogicNode* pSrc, SWindowLogicNode* p CLONE_NODE_FIELD(pTspk); CLONE_NODE_FIELD(pTsEnd); CLONE_NODE_FIELD(pStateExpr); + CLONE_NODE_FIELD(pStartCond); + CLONE_NODE_FIELD(pEndCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(deleteMark); @@ -712,6 +721,9 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_STATE_WINDOW: code = stateWindowNodeCopy((const SStateWindowNode*)pNode, (SStateWindowNode*)pDst); break; + case QUERY_NODE_EVENT_WINDOW: + code = eventWindowNodeCopy((const SEventWindowNode*)pNode, (SEventWindowNode*)pDst); + break; case QUERY_NODE_SESSION_WINDOW: code = sessionWindowNodeCopy((const SSessionWindowNode*)pNode, (SSessionWindowNode*)pDst); break; diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c19f64bb3a0051afc72d8e821c2c0962b363148d..099cd0d3b3adf1a7404fa2cbd947bfc1800fd9c3 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -79,17 +79,23 @@ const char* nodesNodeName(ENodeType type) { return "TableOptions"; case QUERY_NODE_INDEX_OPTIONS: return "IndexOptions"; + case QUERY_NODE_EXPLAIN_OPTIONS: + return "ExplainOptions"; + case QUERY_NODE_STREAM_OPTIONS: + return "StreamOptions"; case QUERY_NODE_LEFT_VALUE: return "LeftValue"; case QUERY_NODE_WHEN_THEN: return "WhenThen"; case QUERY_NODE_CASE_WHEN: return "CaseWhen"; + case QUERY_NODE_EVENT_WINDOW: + return "EventWindow"; case QUERY_NODE_SET_OPERATOR: return "SetOperator"; case QUERY_NODE_SELECT_STMT: return "SelectStmt"; - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: return "VnodeModifStmt"; case QUERY_NODE_CREATE_DATABASE_STMT: return "CreateDatabaseStmt"; @@ -97,11 +103,15 @@ const char* nodesNodeName(ENodeType type) { return "DropDatabaseStmt"; case QUERY_NODE_ALTER_DATABASE_STMT: return "AlterDatabaseStmt"; + case QUERY_NODE_FLUSH_DATABASE_STMT: + return "FlushDatabaseStmt"; + case QUERY_NODE_TRIM_DATABASE_STMT: + return "TrimDatabaseStmt"; case QUERY_NODE_CREATE_TABLE_STMT: return "CreateTableStmt"; case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: return "CreateSubtableClause"; - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return "CreateMultiTableStmt"; case QUERY_NODE_DROP_TABLE_CLAUSE: return "DropTableClause"; @@ -135,12 +145,48 @@ const char* nodesNodeName(ENodeType type) { return "CreateQnodeStmt"; case QUERY_NODE_DROP_QNODE_STMT: return "DropQnodeStmt"; + case QUERY_NODE_CREATE_SNODE_STMT: + return "CreateSnodeStmt"; + case QUERY_NODE_DROP_SNODE_STMT: + return "DropSnodeStmt"; + case QUERY_NODE_CREATE_MNODE_STMT: + return "CreateMnodeStmt"; + case QUERY_NODE_DROP_MNODE_STMT: + return "DropMnodeStmt"; case QUERY_NODE_CREATE_TOPIC_STMT: return "CreateTopicStmt"; case QUERY_NODE_DROP_TOPIC_STMT: return "DropTopicStmt"; + case QUERY_NODE_DROP_CGROUP_STMT: + return "DropConsumerGroupStmt"; case QUERY_NODE_ALTER_LOCAL_STMT: return "AlterLocalStmt"; + case QUERY_NODE_EXPLAIN_STMT: + return "ExplainStmt"; + case QUERY_NODE_DESCRIBE_STMT: + return "DescribeStmt"; + case QUERY_NODE_COMPACT_DATABASE_STMT: + return "CompactDatabaseStmt"; + case QUERY_NODE_CREATE_STREAM_STMT: + return "CreateStreamStmt"; + case QUERY_NODE_DROP_STREAM_STMT: + return "DropStreamStmt"; + case QUERY_NODE_BALANCE_VGROUP_STMT: + return "BalanceVgroupStmt"; + case QUERY_NODE_MERGE_VGROUP_STMT: + return "MergeVgroupStmt"; + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + return "ShowDbAliveStmt"; + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return "ShowClusterAliveStmt"; + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return "RedistributeVgroupStmt"; + case QUERY_NODE_SPLIT_VGROUP_STMT: + return "SplitVgroupStmt"; + case QUERY_NODE_GRANT_STMT: + return "GrantStmt"; + case QUERY_NODE_REVOKE_STMT: + return "RevokeStmt"; case QUERY_NODE_SHOW_DNODES_STMT: return "ShowDnodesStmt"; case QUERY_NODE_SHOW_MNODES_STMT: @@ -153,6 +199,8 @@ const char* nodesNodeName(ENodeType type) { return "ShowSnodesStmt"; case QUERY_NODE_SHOW_BNODES_STMT: return "ShowBnodesStmt"; + case QUERY_NODE_SHOW_CLUSTER_STMT: + return "ShowClusterStmt"; case QUERY_NODE_SHOW_DATABASES_STMT: return "ShowDatabaseStmt"; case QUERY_NODE_SHOW_FUNCTIONS_STMT: @@ -179,8 +227,30 @@ const char* nodesNodeName(ENodeType type) { return "ShowConsumersStmt"; case QUERY_NODE_SHOW_QUERIES_STMT: return "ShowQueriesStmt"; + case QUERY_NODE_SHOW_VARIABLES_STMT: + return "ShowVariablesStmt"; + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return "ShowDnodeVariablesStmt"; + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return "ShowTransactionsStmt"; + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return "ShowSubscriptionsStmt"; case QUERY_NODE_SHOW_VNODES_STMT: return "ShowVnodeStmt"; + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return "ShowUserPrivilegesStmt"; + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return "ShowCreateDatabasesStmt"; + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return "ShowCreateTablesStmt"; + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return "ShowCreateStablesStmt"; + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return "ShowTableDistributedStmt"; + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return "ShowLocalVariablesStmt"; + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return "ShowTableTagsStmt"; case QUERY_NODE_DELETE_STMT: return "DeleteStmt"; case QUERY_NODE_INSERT_STMT: @@ -233,6 +303,10 @@ const char* nodesNodeName(ENodeType type) { return "PhysiLastRowScan"; case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: return "PhysiTableCountScan"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return "PhysiMergeEventWindow"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return "PhysiStreamEventWindow"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -1358,6 +1432,23 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) { + SJoinLogicNode* pNode = (SJoinLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkJoinLogicPlanJoinType, pNode->joinType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanMergeCondition, &pNode->pMergeCondition); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOnConditions); + } + + return code; +} + static const char* jkPhysiPlanOutputDataBlockDesc = "OutputDataBlockDesc"; static const char* jkPhysiPlanConditions = "Conditions"; static const char* jkPhysiPlanChildren = "Children"; @@ -2286,6 +2377,37 @@ static int32_t jsonToPhysiStateWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowPhysiPlanStartCond = "StartCond"; +static const char* jkEventWindowPhysiPlanEndCond = "EndCond"; + +static int32_t physiEventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = physiWindowNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowPhysiPlanEndCond, nodeToJson, pNode->pEndCond); + } + + return code; +} + +static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = jsonToPhysiWindowNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowPhysiPlanEndCond, &pNode->pEndCond); + } + + return code; +} + static const char* jkPartitionPhysiPlanExprs = "Exprs"; static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys"; static const char* jkPartitionPhysiPlanTargets = "Targets"; @@ -2482,6 +2604,7 @@ static const char* jkQueryInsertPhysiPlanTableType = "TableType"; static const char* jkQueryInsertPhysiPlanTableFName = "TableFName"; static const char* jkQueryInsertPhysiPlanVgId = "VgId"; static const char* jkQueryInsertPhysiPlanEpSet = "EpSet"; +static const char* jkQueryInsertPhysiPlanExplain = "Explain"; static int32_t physiQueryInsertNodeToJson(const void* pObj, SJson* pJson) { const SQueryInserterNode* pNode = (const SQueryInserterNode*)pObj; @@ -2508,6 +2631,9 @@ static int32_t physiQueryInsertNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkQueryInsertPhysiPlanEpSet, epSetToJson, &pNode->epSet); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkQueryInsertPhysiPlanExplain, pNode->explain); + } return code; } @@ -2537,6 +2663,9 @@ static int32_t jsonToPhysiQueryInsertNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkQueryInsertPhysiPlanEpSet, jsonToEpSet, &pNode->epSet); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkQueryInsertPhysiPlanExplain, &pNode->explain); + } return code; } @@ -3546,6 +3675,51 @@ static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkJoinTableJoinType = "JoinType"; +static const char* jkJoinTableLeft = "Left"; +static const char* jkJoinTableRight = "Right"; +static const char* jkJoinTableOnCond = "OnCond"; + +static int32_t joinTableNodeToJson(const void* pObj, SJson* pJson) { + const SJoinTableNode* pNode = (const SJoinTableNode*)pObj; + + int32_t code = tableNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkJoinTableJoinType, pNode->joinType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableLeft, nodeToJson, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableRight, nodeToJson, pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinTableOnCond, nodeToJson, pNode->pOnCond); + } + + return code; +} + +static int32_t jsonToJoinTableNode(const SJson* pJson, void* pObj) { + SJoinTableNode* pNode = (SJoinTableNode*)pObj; + + int32_t code = jsonToTableNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkJoinTableJoinType, pNode->joinType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableLeft, &pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableRight, &pNode->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinTableOnCond, &pNode->pOnCond); + } + + return code; +} + static const char* jkGroupingSetType = "GroupingSetType"; static const char* jkGroupingSetParameter = "Parameters"; @@ -3674,6 +3848,36 @@ static int32_t jsonToSessionWindowNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkEventWindowTsPrimaryKey = "TsPrimaryKey"; +static const char* jkEventWindowStartCond = "StartCond"; +static const char* jkEventWindowEndCond = "EndCond"; + +static int32_t eventWindowNodeToJson(const void* pObj, SJson* pJson) { + const SEventWindowNode* pNode = (const SEventWindowNode*)pObj; + + int32_t code = tjsonAddObject(pJson, jkEventWindowTsPrimaryKey, nodeToJson, pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowStartCond, nodeToJson, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkEventWindowEndCond, nodeToJson, pNode->pEndCond); + } + return code; +} + +static int32_t jsonToEventWindowNode(const SJson* pJson, void* pObj) { + SEventWindowNode* pNode = (SEventWindowNode*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkEventWindowTsPrimaryKey, &pNode->pCol); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowStartCond, &pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkEventWindowEndCond, &pNode->pEndCond); + } + return code; +} + static const char* jkIntervalWindowInterval = "Interval"; static const char* jkIntervalWindowOffset = "Offset"; static const char* jkIntervalWindowSliding = "Sliding"; @@ -3870,6 +4074,45 @@ static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkColumnDefColName = "ColName"; +static const char* jkColumnDefDataType = "DataType"; +static const char* jkColumnDefComments = "Comments"; +static const char* jkColumnDefSma = "Sma"; + +static int32_t columnDefNodeToJson(const void* pObj, SJson* pJson) { + const SColumnDefNode* pNode = (const SColumnDefNode*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkColumnDefColName, pNode->colName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkColumnDefDataType, dataTypeToJson, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkColumnDefComments, pNode->comments); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkColumnDefSma, pNode->sma); + } + + return code; +} + +static int32_t jsonToColumnDefNode(const SJson* pJson, void* pObj) { + SColumnDefNode* pNode = (SColumnDefNode*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkColumnDefColName, pNode->colName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkColumnDefDataType, jsonToDataType, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnDefComments, pNode->comments); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkColumnDefSma, &pNode->sma); + } + + return code; +} + static const char* jkDownstreamSourceAddr = "Addr"; static const char* jkDownstreamSourceTaskId = "TaskId"; static const char* jkDownstreamSourceSchedId = "SchedId"; @@ -4060,6 +4303,190 @@ static int32_t jsonToDatabaseOptions(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableOptionsComment = "Comment"; +static const char* jkTableOptionsMaxDelay = "MaxDelay"; +static const char* jkTableOptionsWatermark = "Watermark"; +static const char* jkTableOptionsDeleteMark = "DeleteMark"; +static const char* jkTableOptionsRollupFuncs = "RollupFuncs"; +static const char* jkTableOptionsTtl = "Ttl"; +static const char* jkTableOptionsSma = "Sma"; + +static int32_t tableOptionsToJson(const void* pObj, SJson* pJson) { + const STableOptions* pNode = (const STableOptions*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkTableOptionsComment, pNode->comment); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsMaxDelay, pNode->pMaxDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsWatermark, pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsDeleteMark, pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsRollupFuncs, pNode->pRollupFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableOptionsTtl, pNode->ttl); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkTableOptionsSma, pNode->pSma); + } + + return code; +} + +static int32_t jsonToTableOptions(const SJson* pJson, void* pObj) { + STableOptions* pNode = (STableOptions*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkTableOptionsComment, pNode->comment); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsMaxDelay, &pNode->pMaxDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsWatermark, &pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsDeleteMark, &pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsRollupFuncs, &pNode->pRollupFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTableOptionsTtl, &pNode->ttl); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkTableOptionsSma, &pNode->pSma); + } + + return code; +} + +static const char* jkIndexOptionsFuncs = "Funcs"; +static const char* jkIndexOptionsInterval = "Interval"; +static const char* jkIndexOptionsOffset = "Offset"; +static const char* jkIndexOptionsSliding = "Sliding"; +static const char* jkIndexOptionsStreamOptions = "StreamOptions"; + +static int32_t indexOptionsToJson(const void* pObj, SJson* pJson) { + const SIndexOptions* pNode = (const SIndexOptions*)pObj; + + int32_t code = nodeListToJson(pJson, jkIndexOptionsFuncs, pNode->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsInterval, nodeToJson, pNode->pInterval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsOffset, nodeToJson, pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsSliding, nodeToJson, pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkIndexOptionsStreamOptions, nodeToJson, pNode->pStreamOptions); + } + + return code; +} + +static int32_t jsonToIndexOptions(const SJson* pJson, void* pObj) { + SIndexOptions* pNode = (SIndexOptions*)pObj; + + int32_t code = jsonToNodeList(pJson, jkIndexOptionsFuncs, &pNode->pFuncs); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsInterval, &pNode->pInterval); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsOffset, &pNode->pOffset); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsSliding, &pNode->pSliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkIndexOptionsStreamOptions, &pNode->pStreamOptions); + } + + return code; +} + +static const char* jkExplainOptionsVerbose = "Verbose"; +static const char* jkExplainOptionsRatio = "Ratio"; + +static int32_t explainOptionsToJson(const void* pObj, SJson* pJson) { + const SExplainOptions* pNode = (const SExplainOptions*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkExplainOptionsVerbose, pNode->verbose); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddDoubleToObject(pJson, jkExplainOptionsRatio, pNode->ratio); + } + + return code; +} + +static int32_t jsonToExplainOptions(const SJson* pJson, void* pObj) { + SExplainOptions* pNode = (SExplainOptions*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkExplainOptionsVerbose, &pNode->verbose); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetDoubleValue(pJson, jkExplainOptionsRatio, &pNode->ratio); + } + + return code; +} + +static const char* jkStreamOptionsTriggerType = "TriggerType"; +static const char* jkStreamOptionsDelay = "Delay"; +static const char* jkStreamOptionsWatermark = "Watermark"; +static const char* jkStreamOptionsDeleteMark = "DeleteMark"; +static const char* jkStreamOptionsFillHistory = "FillHistory"; +static const char* jkStreamOptionsIgnoreExpired = "IgnoreExpired"; + +static int32_t streamOptionsToJson(const void* pObj, SJson* pJson) { + const SStreamOptions* pNode = (const SStreamOptions*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkStreamOptionsTriggerType, pNode->triggerType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsDelay, nodeToJson, pNode->pDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsWatermark, nodeToJson, pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkStreamOptionsDeleteMark, nodeToJson, pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkStreamOptionsFillHistory, pNode->fillHistory); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkStreamOptionsIgnoreExpired, pNode->ignoreExpired); + } + + return code; +} + +static int32_t jsonToStreamOptions(const SJson* pJson, void* pObj) { + SStreamOptions* pNode = (SStreamOptions*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkStreamOptionsTriggerType, &pNode->triggerType); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsDelay, &pNode->pDelay); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsWatermark, &pNode->pWatermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkStreamOptionsDeleteMark, &pNode->pDeleteMark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkStreamOptionsFillHistory, &pNode->fillHistory); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkStreamOptionsIgnoreExpired, &pNode->ignoreExpired); + } + + return code; +} + static const char* jkWhenThenWhen = "When"; static const char* jkWhenThenThen = "Then"; @@ -4345,164 +4772,1414 @@ static int32_t jsonToSelectStmt(const SJson* pJson, void* pObj) { return code; } -static const char* jkAlterDatabaseStmtDbName = "DbName"; -static const char* jkAlterDatabaseStmtOptions = "Options"; +static const char* jkVnodeModifyOpStmtSqlNodeType = "SqlNodeType"; +static const char* jkVnodeModifyOpStmtTotalRowsNum = "TotalRowsNum"; +static const char* jkVnodeModifyOpStmtTotalTbNum = "TotalTbNum"; -static int32_t alterDatabaseStmtToJson(const void* pObj, SJson* pJson) { - const SAlterDatabaseStmt* pNode = (const SAlterDatabaseStmt*)pObj; +static int32_t vnodeModifyStmtToJson(const void* pObj, SJson* pJson) { + const SVnodeModifyOpStmt* pNode = (const SVnodeModifyOpStmt*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + int32_t code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtSqlNodeType, pNode->sqlNodeType); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterDatabaseStmtOptions, nodeToJson, pNode->pOptions); + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtTotalRowsNum, pNode->totalRowsNum); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkVnodeModifyOpStmtTotalTbNum, pNode->totalTbNum); } return code; } -static int32_t jsonToAlterDatabaseStmt(const SJson* pJson, void* pObj) { - SAlterDatabaseStmt* pNode = (SAlterDatabaseStmt*)pObj; +static int32_t jsonToVnodeModifyStmt(const SJson* pJson, void* pObj) { + SVnodeModifyOpStmt* pNode = (SVnodeModifyOpStmt*)pObj; - int32_t code = tjsonGetStringValue(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkVnodeModifyOpStmtSqlNodeType, pNode->sqlNodeType, code); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterDatabaseStmtOptions, (SNode**)&pNode->pOptions); + code = tjsonGetIntValue(pJson, jkVnodeModifyOpStmtTotalRowsNum, &pNode->totalRowsNum); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkVnodeModifyOpStmtTotalTbNum, &pNode->totalTbNum); } return code; } -static const char* jkAlterTableStmtDbName = "DbName"; -static const char* jkAlterTableStmtTableName = "TableName"; -static const char* jkAlterTableStmtAlterType = "AlterType"; -static const char* jkAlterTableStmtColName = "ColName"; -static const char* jkAlterTableStmtNewColName = "NewColName"; -static const char* jkAlterTableStmtOptions = "Options"; -static const char* jkAlterTableStmtNewDataType = "NewDataType"; -static const char* jkAlterTableStmtNewTagVal = "NewTagVal"; +static const char* jkCreateDatabaseStmtDbName = "DbName"; +static const char* jkCreateDatabaseStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateDatabaseStmtOptions = "Options"; -static int32_t alterTableStmtToJson(const void* pObj, SJson* pJson) { - const SAlterTableStmt* pNode = (const SAlterTableStmt*)pObj; +static int32_t createDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SCreateDatabaseStmt* pNode = (const SCreateDatabaseStmt*)pObj; - int32_t code = tjsonAddStringToObject(pJson, jkAlterTableStmtDbName, pNode->dbName); + int32_t code = tjsonAddStringToObject(pJson, jkCreateDatabaseStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtTableName, pNode->tableName); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkAlterTableStmtAlterType, pNode->alterType); + code = tjsonAddBoolToObject(pJson, jkCreateDatabaseStmtIgnoreExists, pNode->ignoreExists); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtColName, pNode->colName); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterTableStmtNewColName, pNode->newColName); + code = tjsonAddObject(pJson, jkCreateDatabaseStmtOptions, nodeToJson, pNode->pOptions); } + + return code; +} + +static int32_t jsonToCreateDatabaseStmt(const SJson* pJson, void* pObj) { + SCreateDatabaseStmt* pNode = (SCreateDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateDatabaseStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pOptions); + code = tjsonGetBoolValue(pJson, jkCreateDatabaseStmtIgnoreExists, &pNode->ignoreExists); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkAlterTableStmtNewDataType, dataTypeToJson, &pNode->dataType); + code = jsonToNodeObject(pJson, jkCreateDatabaseStmtOptions, (SNode**)&pNode->pOptions); } - if (TSDB_CODE_SUCCESS == code) { + + return code; +} + +static const char* jkAlterDatabaseStmtDbName = "DbName"; +static const char* jkAlterDatabaseStmtOptions = "Options"; + +static int32_t alterDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SAlterDatabaseStmt* pNode = (const SAlterDatabaseStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterDatabaseStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToAlterDatabaseStmt(const SJson* pJson, void* pObj) { + SAlterDatabaseStmt* pNode = (SAlterDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterDatabaseStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkTrimDatabaseStmtDbName = "DbName"; +static const char* jkTrimDatabaseStmtMaxSpeed = "MaxSpeed"; + +static int32_t trimDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const STrimDatabaseStmt* pNode = (const STrimDatabaseStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkTrimDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTrimDatabaseStmtMaxSpeed, pNode->maxSpeed); + } + + return code; +} + +static int32_t jsonToTrimDatabaseStmt(const SJson* pJson, void* pObj) { + STrimDatabaseStmt* pNode = (STrimDatabaseStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkTrimDatabaseStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTrimDatabaseStmtMaxSpeed, &pNode->maxSpeed); + } + + return code; +} + +static const char* jkCreateTableStmtDbName = "DbName"; +static const char* jkCreateTableStmtTableName = "TableName"; +static const char* jkCreateTableStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTableStmtCols = "Cols"; +static const char* jkCreateTableStmtTags = "Tags"; +static const char* jkCreateTableStmtOptions = "Options"; + +static int32_t createTableStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTableStmt* pNode = (const SCreateTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTableStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateTableStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateTableStmtTags, pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTableStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateTableStmt(const SJson* pJson, void* pObj) { + SCreateTableStmt* pNode = (SCreateTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTableStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateTableStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateTableStmtTags, &pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTableStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkCreateSubTableClauseDbName = "DbName"; +static const char* jkCreateSubTableClauseTableName = "TableName"; +static const char* jkCreateSubTableClauseUseDbName = "UseDbName"; +static const char* jkCreateSubTableClauseUseTableName = "UseTableName"; +static const char* jkCreateSubTableClauseIgnoreExists = "IgnoreExists"; +static const char* jkCreateSubTableClauseSpecificTags = "SpecificTags"; +static const char* jkCreateSubTableClauseValsOfTags = "ValsOfTags"; +static const char* jkCreateSubTableClauseOptions = "Options"; + +static int32_t createSubTableClauseToJson(const void* pObj, SJson* pJson) { + const SCreateSubTableClause* pNode = (const SCreateSubTableClause*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseUseDbName, pNode->useDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateSubTableClauseUseTableName, pNode->useTableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateSubTableClauseIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateSubTableClauseSpecificTags, pNode->pSpecificTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateSubTableClauseValsOfTags, pNode->pValsOfTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateSubTableClauseOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateSubTableClause(const SJson* pJson, void* pObj) { + SCreateSubTableClause* pNode = (SCreateSubTableClause*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateSubTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseUseDbName, pNode->useDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateSubTableClauseUseTableName, pNode->useTableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateSubTableClauseIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateSubTableClauseSpecificTags, &pNode->pSpecificTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateSubTableClauseValsOfTags, &pNode->pValsOfTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateSubTableClauseOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkCreateMultiTablesStmtSubTables = "SubTables"; + +static int32_t createMultiTablesStmtToJson(const void* pObj, SJson* pJson) { + const SCreateMultiTablesStmt* pNode = (const SCreateMultiTablesStmt*)pObj; + return nodeListToJson(pJson, jkCreateMultiTablesStmtSubTables, pNode->pSubTables); +} + +static int32_t jsonToCreateMultiTablesStmt(const SJson* pJson, void* pObj) { + SCreateMultiTablesStmt* pNode = (SCreateMultiTablesStmt*)pObj; + return jsonToNodeList(pJson, jkCreateMultiTablesStmtSubTables, &pNode->pSubTables); +} + +static const char* jkDropTableClauseDbName = "DbName"; +static const char* jkDropTableClauseTableName = "TableName"; +static const char* jkDropTableClauseIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropTableClauseToJson(const void* pObj, SJson* pJson) { + const SDropTableClause* pNode = (const SDropTableClause*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropTableClauseIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropTableClause(const SJson* pJson, void* pObj) { + SDropTableClause* pNode = (SDropTableClause*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropTableClauseDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropTableClauseTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropTableClauseIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkDropTableStmtTables = "Tables"; + +static int32_t dropTableStmtToJson(const void* pObj, SJson* pJson) { + const SDropTableStmt* pNode = (const SDropTableStmt*)pObj; + return nodeListToJson(pJson, jkDropTableStmtTables, pNode->pTables); +} + +static int32_t jsonToDropTableStmt(const SJson* pJson, void* pObj) { + SDropTableStmt* pNode = (SDropTableStmt*)pObj; + return jsonToNodeList(pJson, jkDropTableStmtTables, &pNode->pTables); +} + +static const char* jkDropSuperTableStmtDbName = "DbName"; +static const char* jkDropSuperTableStmtTableName = "TableName"; +static const char* jkDropSuperTableStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropStableStmtToJson(const void* pObj, SJson* pJson) { + const SDropSuperTableStmt* pNode = (const SDropSuperTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropSuperTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropSuperTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropSuperTableStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropStableStmt(const SJson* pJson, void* pObj) { + SDropSuperTableStmt* pNode = (SDropSuperTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropSuperTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropSuperTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropSuperTableStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkAlterTableStmtDbName = "DbName"; +static const char* jkAlterTableStmtTableName = "TableName"; +static const char* jkAlterTableStmtAlterType = "AlterType"; +static const char* jkAlterTableStmtColName = "ColName"; +static const char* jkAlterTableStmtNewColName = "NewColName"; +static const char* jkAlterTableStmtOptions = "Options"; +static const char* jkAlterTableStmtNewDataType = "NewDataType"; +static const char* jkAlterTableStmtNewTagVal = "NewTagVal"; + +static int32_t alterTableStmtToJson(const void* pObj, SJson* pJson) { + const SAlterTableStmt* pNode = (const SAlterTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterTableStmtAlterType, pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtColName, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterTableStmtNewColName, pNode->newColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkAlterTableStmtNewDataType, dataTypeToJson, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkAlterTableStmtOptions, nodeToJson, pNode->pVal); } return code; } -static int32_t jsonToAlterTableStmt(const SJson* pJson, void* pObj) { - SAlterTableStmt* pNode = (SAlterTableStmt*)pObj; +static int32_t jsonToAlterTableStmt(const SJson* pJson, void* pObj) { + SAlterTableStmt* pNode = (SAlterTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterTableStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterTableStmtAlterType, &pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtColName, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterTableStmtNewColName, pNode->newColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkAlterTableStmtNewDataType, jsonToDataType, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pVal); + } + + return code; +} + +static int32_t alterStableStmtToJson(const void* pObj, SJson* pJson) { return alterTableStmtToJson(pObj, pJson); } + +static int32_t jsonToAlterStableStmt(const SJson* pJson, void* pObj) { return jsonToAlterTableStmt(pJson, pObj); } + +static const char* jkCreateUserStmtUserName = "UserName"; +static const char* jkCreateUserStmtPassword = "Password"; +static const char* jkCreateUserStmtSysinfo = "Sysinfo"; + +static int32_t createUserStmtToJson(const void* pObj, SJson* pJson) { + const SCreateUserStmt* pNode = (const SCreateUserStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkCreateUserStmtSysinfo, pNode->sysinfo); + } + + return code; +} + +static int32_t jsonToCreateUserStmt(const SJson* pJson, void* pObj) { + SCreateUserStmt* pNode = (SCreateUserStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkCreateUserStmtSysinfo, &pNode->sysinfo); + } + + return code; +} + +static const char* jkAlterUserStmtUserName = "UserName"; +static const char* jkAlterUserStmtAlterType = "AlterType"; +static const char* jkAlterUserStmtPassword = "Password"; +static const char* jkAlterUserStmtEnable = "Enable"; +static const char* jkAlterUserStmtSysinfo = "Sysinfo"; + +static int32_t alterUserStmtToJson(const void* pObj, SJson* pJson) { + const SAlterUserStmt* pNode = (const SAlterUserStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtAlterType, pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtEnable, pNode->enable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkAlterUserStmtSysinfo, pNode->sysinfo); + } + + return code; +} + +static int32_t jsonToAlterUserStmt(const SJson* pJson, void* pObj) { + SAlterUserStmt* pNode = (SAlterUserStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterUserStmtUserName, pNode->userName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtAlterType, &pNode->alterType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterUserStmtPassword, pNode->password); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtEnable, &pNode->enable); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkAlterUserStmtSysinfo, &pNode->sysinfo); + } + + return code; +} + +static const char* jkDropUserStmtUserName = "UserName"; + +static int32_t dropUserStmtToJson(const void* pObj, SJson* pJson) { + const SDropUserStmt* pNode = (const SDropUserStmt*)pObj; + return tjsonAddStringToObject(pJson, jkDropUserStmtUserName, pNode->userName); +} + +static int32_t jsonToDropUserStmt(const SJson* pJson, void* pObj) { + SDropUserStmt* pNode = (SDropUserStmt*)pObj; + return tjsonGetStringValue(pJson, jkDropUserStmtUserName, pNode->userName); +} + +static const char* jkUseDatabaseStmtDbName = "DbName"; + +static int32_t useDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SUseDatabaseStmt* pNode = (const SUseDatabaseStmt*)pObj; + return tjsonAddStringToObject(pJson, jkUseDatabaseStmtDbName, pNode->dbName); +} + +static int32_t jsonToUseDatabaseStmt(const SJson* pJson, void* pObj) { + SUseDatabaseStmt* pNode = (SUseDatabaseStmt*)pObj; + return tjsonGetStringValue(pJson, jkUseDatabaseStmtDbName, pNode->dbName); +} + +static const char* jkCreateDnodeStmtFqdn = "Fqdn"; +static const char* jkCreateDnodeStmtPort = "Port"; + +static int32_t createDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SCreateDnodeStmt* pNode = (const SCreateDnodeStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateDnodeStmtFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkCreateDnodeStmtPort, pNode->port); + } + + return code; +} + +static int32_t jsonToCreateDnodeStmt(const SJson* pJson, void* pObj) { + SCreateDnodeStmt* pNode = (SCreateDnodeStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateDnodeStmtFqdn, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkCreateDnodeStmtPort, &pNode->port); + } + + return code; +} + +static const char* jkAlterDnodeStmtDnodeId = "DnodeId"; +static const char* jkAlterDnodeStmtConfig = "Config"; +static const char* jkAlterDnodeStmtValue = "Value"; + +static int32_t alterDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SAlterDnodeStmt* pNode = (const SAlterDnodeStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkAlterDnodeStmtDnodeId, pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtConfig, pNode->config); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtValue, pNode->value); + } + + return code; +} + +static int32_t jsonToAlterDnodeStmt(const SJson* pJson, void* pObj) { + SAlterDnodeStmt* pNode = (SAlterDnodeStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkAlterDnodeStmtDnodeId, &pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterDnodeStmtConfig, pNode->config); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterDnodeStmtValue, pNode->value); + } + + return code; +} + +static const char* jkCreateIndexStmtIndexType = "IndexType"; +static const char* jkCreateIndexStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateIndexStmtIndexDbName = "IndexDbName"; +static const char* jkCreateIndexStmtIndexName = "indexName"; +static const char* jkCreateIndexStmtDbName = "DbName"; +static const char* jkCreateIndexStmtTableName = "TableName"; +static const char* jkCreateIndexStmtCols = "Cols"; +static const char* jkCreateIndexStmtOptions = "Options"; + +static int32_t createIndexStmtToJson(const void* pObj, SJson* pJson) { + const SCreateIndexStmt* pNode = (const SCreateIndexStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkCreateIndexStmtIndexType, pNode->indexType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateIndexStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtIndexName, pNode->indexName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateIndexStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateIndexStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateIndexStmtOptions, nodeToJson, pNode->pOptions); + } + + return code; +} + +static int32_t jsonToCreateIndexStmt(const SJson* pJson, void* pObj) { + SCreateIndexStmt* pNode = (SCreateIndexStmt*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + tjsonGetNumberValue(pJson, jkCreateIndexStmtIndexType, pNode->indexType, code); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateIndexStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtIndexName, pNode->indexName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateIndexStmtTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateIndexStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateIndexStmtOptions, (SNode**)&pNode->pOptions); + } + + return code; +} + +static const char* jkDropIndexStmtIgnoreNotExists = "IgnoreNotExists"; +static const char* jkDropIndexStmtIndexDbName = "IndexDbName"; +static const char* jkDropIndexStmtIndexName = "IndexName"; + +static int32_t dropIndexStmtToJson(const void* pObj, SJson* pJson) { + const SDropIndexStmt* pNode = (const SDropIndexStmt*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkDropIndexStmtIgnoreNotExists, pNode->ignoreNotExists); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropIndexStmtIndexName, pNode->indexName); + } + + return code; +} + +static int32_t jsonToDropIndexStmt(const SJson* pJson, void* pObj) { + SDropIndexStmt* pNode = (SDropIndexStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkDropIndexStmtIgnoreNotExists, &pNode->ignoreNotExists); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropIndexStmtIndexDbName, pNode->indexDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropIndexStmtIndexName, pNode->indexName); + } + + return code; +} + +static const char* jkCreateComponentNodeStmtDnodeId = "DnodeId"; + +static int32_t createComponentNodeStmtToJson(const void* pObj, SJson* pJson) { + const SCreateComponentNodeStmt* pNode = (const SCreateComponentNodeStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkCreateComponentNodeStmtDnodeId, pNode->dnodeId); +} + +static int32_t jsonToCreateComponentNodeStmt(const SJson* pJson, void* pObj) { + SCreateComponentNodeStmt* pNode = (SCreateComponentNodeStmt*)pObj; + return tjsonGetIntValue(pJson, jkCreateComponentNodeStmtDnodeId, &pNode->dnodeId); +} + +static const char* jkDropComponentNodeStmtDnodeId = "DnodeId"; + +static int32_t dropComponentNodeStmtToJson(const void* pObj, SJson* pJson) { + const SDropComponentNodeStmt* pNode = (const SDropComponentNodeStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkDropComponentNodeStmtDnodeId, pNode->dnodeId); +} + +static int32_t jsonToDropComponentNodeStmt(const SJson* pJson, void* pObj) { + SDropComponentNodeStmt* pNode = (SDropComponentNodeStmt*)pObj; + return tjsonGetIntValue(pJson, jkDropComponentNodeStmtDnodeId, &pNode->dnodeId); +} + +static int32_t createQnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateQnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropQnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropQnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static int32_t createSnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateSnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropSnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropSnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static int32_t createMnodeStmtToJson(const void* pObj, SJson* pJson) { + return createComponentNodeStmtToJson(pObj, pJson); +} + +static int32_t jsonToCreateMnodeStmt(const SJson* pJson, void* pObj) { + return jsonToCreateComponentNodeStmt(pJson, pObj); +} + +static int32_t dropMnodeStmtToJson(const void* pObj, SJson* pJson) { return dropComponentNodeStmtToJson(pObj, pJson); } + +static int32_t jsonToDropMnodeStmt(const SJson* pJson, void* pObj) { return jsonToDropComponentNodeStmt(pJson, pObj); } + +static const char* jkDropDnodeStmtDnodeId = "DnodeId"; +static const char* jkDropDnodeStmtFqdn = "Fqdn"; +static const char* jkDropDnodeStmtPort = "Port"; +static const char* jkDropDnodeStmtForce = "Force"; + +static int32_t dropDnodeStmtToJson(const void* pObj, SJson* pJson) { + const SDropDnodeStmt* pNode = (const SDropDnodeStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkDropDnodeStmtDnodeId, pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropDnodeStmtFqdn, pNode->fqdn); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkDropDnodeStmtPort, pNode->port); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropDnodeStmtForce, pNode->force); + } + + return code; +} + +static int32_t jsonToDropDnodeStmt(const SJson* pJson, void* pObj) { + SDropDnodeStmt* pNode = (SDropDnodeStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkDropDnodeStmtDnodeId, &pNode->dnodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropDnodeStmtFqdn, pNode->fqdn); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDropDnodeStmtPort, &pNode->port); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropDnodeStmtForce, &pNode->force); + } + + return code; +} + +static const char* jkCreateTopicStmtTopicName = "TopicName"; +static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; +static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateTopicStmtQuery = "Query"; + +static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { + const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { + SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + } + + return code; +} + +static const char* jkDropTopicStmtTopicName = "TopicName"; +static const char* jkDropTopicStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropTopicStmtToJson(const void* pObj, SJson* pJson) { + const SDropTopicStmt* pNode = (const SDropTopicStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropTopicStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropTopicStmt(const SJson* pJson, void* pObj) { + SDropTopicStmt* pNode = (SDropTopicStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropTopicStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropTopicStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkDropCGroupStmtTopicName = "TopicName"; +static const char* jkDropCGroupStmtConsumerGroup = "ConsumerGroup"; +static const char* jkDropCGroupStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropConsumerGroupStmtToJson(const void* pObj, SJson* pJson) { + const SDropCGroupStmt* pNode = (const SDropCGroupStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropCGroupStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDropCGroupStmtConsumerGroup, pNode->cgroup); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropCGroupStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropConsumerGroupStmt(const SJson* pJson, void* pObj) { + SDropCGroupStmt* pNode = (SDropCGroupStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropCGroupStmtTopicName, pNode->topicName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDropCGroupStmtConsumerGroup, pNode->cgroup); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropCGroupStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkAlterLocalStmtConfig = "Config"; +static const char* jkAlterLocalStmtValue = "Value"; + +static int32_t alterLocalStmtToJson(const void* pObj, SJson* pJson) { + const SAlterLocalStmt* pNode = (const SAlterLocalStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkAlterLocalStmtConfig, pNode->config); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkAlterLocalStmtValue, pNode->value); + } + + return code; +} + +static int32_t jsonToAlterLocalStmt(const SJson* pJson, void* pObj) { + SAlterLocalStmt* pNode = (SAlterLocalStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkAlterLocalStmtConfig, pNode->config); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkAlterLocalStmtValue, pNode->value); + } + + return code; +} + +static const char* jkExplainStmtAnalyze = "Analyze"; +static const char* jkExplainStmtOptions = "Options"; +static const char* jkExplainStmtQuery = "Query"; + +static int32_t explainStmtToJson(const void* pObj, SJson* pJson) { + const SExplainStmt* pNode = (const SExplainStmt*)pObj; + + int32_t code = tjsonAddBoolToObject(pJson, jkExplainStmtAnalyze, pNode->analyze); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkExplainStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkExplainStmtQuery, nodeToJson, pNode->pQuery); + } + + return code; +} + +static int32_t jsonToExplainStmt(const SJson* pJson, void* pObj) { + SExplainStmt* pNode = (SExplainStmt*)pObj; + + int32_t code = tjsonGetBoolValue(pJson, jkExplainStmtAnalyze, &pNode->analyze); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkExplainStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkExplainStmtQuery, &pNode->pQuery); + } + + return code; +} + +static const char* jkDescribeStmtDbName = "DbName"; +static const char* jkDescribeStmtTableName = "TableName"; + +static int32_t describeStmtToJson(const void* pObj, SJson* pJson) { + const SDescribeStmt* pNode = (const SDescribeStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDescribeStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkDescribeStmtTableName, pNode->tableName); + } + + return code; +} + +static int32_t jsonToDescribeStmt(const SJson* pJson, void* pObj) { + SDescribeStmt* pNode = (SDescribeStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDescribeStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkDescribeStmtTableName, pNode->tableName); + } + + return code; +} + +static const char* jkCompactDatabaseStmtDbName = "DbName"; + +static int32_t compactDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SCompactDatabaseStmt* pNode = (const SCompactDatabaseStmt*)pObj; + return tjsonAddStringToObject(pJson, jkCompactDatabaseStmtDbName, pNode->dbName); +} + +static int32_t jsonToCompactDatabaseStmt(const SJson* pJson, void* pObj) { + SCompactDatabaseStmt* pNode = (SCompactDatabaseStmt*)pObj; + return tjsonGetStringValue(pJson, jkCompactDatabaseStmtDbName, pNode->dbName); +} + +static const char* jkCreateStreamStmtStreamName = "StreamName"; +static const char* jkCreateStreamStmtTargetDbName = "TargetDbName"; +static const char* jkCreateStreamStmtTargetTabName = "TargetTabName"; +static const char* jkCreateStreamStmtIgnoreExists = "IgnoreExists"; +static const char* jkCreateStreamStmtOptions = "Options"; +static const char* jkCreateStreamStmtQuery = "Query"; +static const char* jkCreateStreamStmtTags = "Tags"; +static const char* jkCreateStreamStmtSubtable = "Subtable"; + +static int32_t createStreamStmtToJson(const void* pObj, SJson* pJson) { + const SCreateStreamStmt* pNode = (const SCreateStreamStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkCreateStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateStreamStmtTargetDbName, pNode->targetDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkCreateStreamStmtTargetTabName, pNode->targetTabName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkCreateStreamStmtIgnoreExists, pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtOptions, nodeToJson, pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtQuery, nodeToJson, pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkCreateStreamStmtTags, pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkCreateStreamStmtSubtable, nodeToJson, pNode->pSubtable); + } + + return code; +} + +static int32_t jsonToCreateStreamStmt(const SJson* pJson, void* pObj) { + SCreateStreamStmt* pNode = (SCreateStreamStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkCreateStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateStreamStmtTargetDbName, pNode->targetDbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkCreateStreamStmtTargetTabName, pNode->targetTabName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkCreateStreamStmtIgnoreExists, &pNode->ignoreExists); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtOptions, (SNode**)&pNode->pOptions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtQuery, &pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkCreateStreamStmtTags, &pNode->pTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkCreateStreamStmtSubtable, &pNode->pSubtable); + } + + return code; +} + +static const char* jkDropStreamStmtStreamName = "StreamName"; +static const char* jkDropStreamStmtIgnoreNotExists = "IgnoreNotExists"; + +static int32_t dropStreamStmtToJson(const void* pObj, SJson* pJson) { + const SDropStreamStmt* pNode = (const SDropStreamStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkDropStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkDropStreamStmtIgnoreNotExists, pNode->ignoreNotExists); + } + + return code; +} + +static int32_t jsonToDropStreamStmt(const SJson* pJson, void* pObj) { + SDropStreamStmt* pNode = (SDropStreamStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkDropStreamStmtStreamName, pNode->streamName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkDropStreamStmtIgnoreNotExists, &pNode->ignoreNotExists); + } + + return code; +} + +static const char* jkMergeVgroupStmtVgroupId1 = "VgroupId1"; +static const char* jkMergeVgroupStmtVgroupId2 = "VgroupId2"; + +static int32_t mergeVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SMergeVgroupStmt* pNode = (const SMergeVgroupStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkMergeVgroupStmtVgroupId1, pNode->vgId1); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergeVgroupStmtVgroupId2, pNode->vgId2); + } + + return code; +} + +static int32_t jsonToMergeVgroupStmt(const SJson* pJson, void* pObj) { + SMergeVgroupStmt* pNode = (SMergeVgroupStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkMergeVgroupStmtVgroupId1, &pNode->vgId1); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergeVgroupStmtVgroupId2, &pNode->vgId2); + } + + return code; +} + +static const char* jkRedistributeVgroupStmtVgroupId = "VgroupId"; +static const char* jkRedistributeVgroupStmtDnodeId1 = "DnodeId1"; +static const char* jkRedistributeVgroupStmtDnodeId2 = "DnodeId2"; +static const char* jkRedistributeVgroupStmtDnodeId3 = "DnodeId3"; +static const char* jkRedistributeVgroupStmtDnodes = "Dnodes"; + +static int32_t redistributeVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SRedistributeVgroupStmt* pNode = (const SRedistributeVgroupStmt*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtVgroupId, pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId1, pNode->dnodeId1); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId2, pNode->dnodeId2); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkRedistributeVgroupStmtDnodeId3, pNode->dnodeId3); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkRedistributeVgroupStmtDnodes, pNode->pDnodes); + } + + return code; +} + +static int32_t jsonToRedistributeVgroupStmt(const SJson* pJson, void* pObj) { + SRedistributeVgroupStmt* pNode = (SRedistributeVgroupStmt*)pObj; + + int32_t code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtVgroupId, &pNode->vgId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId1, &pNode->dnodeId1); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId2, &pNode->dnodeId2); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkRedistributeVgroupStmtDnodeId3, &pNode->dnodeId3); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkRedistributeVgroupStmtDnodes, &pNode->pDnodes); + } + + return code; +} + +static const char* jkSplitVgroupStmtVgroupId = "VgroupId"; - int32_t code = tjsonGetStringValue(pJson, jkAlterTableStmtDbName, pNode->dbName); +static int32_t splitVgroupStmtToJson(const void* pObj, SJson* pJson) { + const SSplitVgroupStmt* pNode = (const SSplitVgroupStmt*)pObj; + return tjsonAddIntegerToObject(pJson, jkSplitVgroupStmtVgroupId, pNode->vgId); +} + +static int32_t jsonToSplitVgroupStmt(const SJson* pJson, void* pObj) { + SSplitVgroupStmt* pNode = (SSplitVgroupStmt*)pObj; + return tjsonGetIntValue(pJson, jkSplitVgroupStmtVgroupId, &pNode->vgId); +} + +static const char* jkGrantStmtUserName = "UserName"; +static const char* jkGrantStmtObjName = "ObjName"; +static const char* jkGrantStmtPrivileges = "Privileges"; + +static int32_t grantStmtToJson(const void* pObj, SJson* pJson) { + const SGrantStmt* pNode = (const SGrantStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkGrantStmtUserName, pNode->userName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtTableName, pNode->tableName); + code = tjsonAddStringToObject(pJson, jkGrantStmtObjName, pNode->objName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetTinyIntValue(pJson, jkAlterTableStmtAlterType, &pNode->alterType); + code = tjsonAddIntegerToObject(pJson, jkGrantStmtPrivileges, pNode->privileges); } + + return code; +} + +static int32_t jsonToGrantStmt(const SJson* pJson, void* pObj) { + SGrantStmt* pNode = (SGrantStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkGrantStmtUserName, pNode->userName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtColName, pNode->colName); + code = tjsonGetStringValue(pJson, jkGrantStmtObjName, pNode->objName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterTableStmtNewColName, pNode->newColName); + code = tjsonGetBigIntValue(pJson, jkGrantStmtPrivileges, &pNode->privileges); } + + return code; +} + +static int32_t revokeStmtToJson(const void* pObj, SJson* pJson) { return grantStmtToJson(pObj, pJson); } + +static int32_t jsonToRevokeStmt(const SJson* pJson, void* pObj) { return jsonToGrantStmt(pJson, pObj); } + +static const char* jkShowStmtDbName = "DbName"; +static const char* jkShowStmtTbName = "TbName"; +static const char* jkShowStmtTableCondType = "TableCondType"; + +static int32_t showStmtToJson(const void* pObj, SJson* pJson) { + const SShowStmt* pNode = (const SShowStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowStmtDbName, nodeToJson, pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pOptions); + code = tjsonAddObject(pJson, jkShowStmtTbName, nodeToJson, pNode->pTbName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonToObject(pJson, jkAlterTableStmtNewDataType, jsonToDataType, &pNode->dataType); + code = tjsonAddIntegerToObject(pJson, jkShowStmtTableCondType, pNode->tableCondType); } + + return code; +} + +static int32_t jsonToShowStmt(const SJson* pJson, void* pObj) { + SShowStmt* pNode = (SShowStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowStmtDbName, &pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkAlterTableStmtOptions, (SNode**)&pNode->pVal); + code = jsonToNodeObject(pJson, jkShowStmtTbName, &pNode->pTbName); + } + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkShowStmtTableCondType, pNode->tableCondType, code); } return code; } -static const char* jkAlterDnodeStmtDnodeId = "DnodeId"; -static const char* jkAlterDnodeStmtConfig = "Config"; -static const char* jkAlterDnodeStmtValue = "Value"; +static int32_t showDnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t alterDnodeStmtToJson(const void* pObj, SJson* pJson) { - const SAlterDnodeStmt* pNode = (const SAlterDnodeStmt*)pObj; +static int32_t jsonToShowDnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } - int32_t code = tjsonAddIntegerToObject(pJson, jkAlterDnodeStmtDnodeId, pNode->dnodeId); +static int32_t showMnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowMnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showQnodesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowQnodesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showClusterStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowClusterStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showDatabasesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowDatabasesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showFunctionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowFunctionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showIndexesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowIndexesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showStablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowStablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showStreamsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowStreamsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showTablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowTablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showTagsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowTagsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showUsersStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowUsersStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showVgroupsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowVgroupsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showConsumersStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowConsumersStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showVariablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowVariablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId"; +static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern"; + +static int32_t showDnodeVariablesStmtToJson(const void* pObj, SJson* pJson) { + const SShowDnodeVariablesStmt* pNode = (const SShowDnodeVariablesStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowDnodeVariablesStmtDnodeId, nodeToJson, pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtConfig, pNode->config); + code = tjsonAddObject(pJson, jkShowDnodeVariablesStmtLikePattern, nodeToJson, pNode->pLikePattern); } + + return code; +} + +static int32_t jsonToShowDnodeVariablesStmt(const SJson* pJson, void* pObj) { + SShowDnodeVariablesStmt* pNode = (SShowDnodeVariablesStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowDnodeVariablesStmtDnodeId, &pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkAlterDnodeStmtValue, pNode->value); + code = jsonToNodeObject(pJson, jkShowDnodeVariablesStmtLikePattern, &pNode->pLikePattern); } return code; } -static int32_t jsonToAlterDnodeStmt(const SJson* pJson, void* pObj) { - SAlterDnodeStmt* pNode = (SAlterDnodeStmt*)pObj; +static int32_t showTransactionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } - int32_t code = tjsonGetIntValue(pJson, jkAlterDnodeStmtDnodeId, &pNode->dnodeId); +static int32_t jsonToShowTransactionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static int32_t showSubscriptionsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } + +static int32_t jsonToShowSubscriptionsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowVnodesStmtDnodeId = "DnodeId"; +static const char* jkShowVnodesStmtDnodeEndpoint = "DnodeEndpoint"; + +static int32_t showVnodesStmtToJson(const void* pObj, SJson* pJson) { + const SShowVnodesStmt* pNode = (const SShowVnodesStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowVnodesStmtDnodeId, nodeToJson, pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterDnodeStmtConfig, pNode->config); + code = tjsonAddObject(pJson, jkShowVnodesStmtDnodeEndpoint, nodeToJson, pNode->pDnodeEndpoint); } + + return code; +} + +static int32_t jsonToShowVnodesStmt(const SJson* pJson, void* pObj) { + SShowVnodesStmt* pNode = (SShowVnodesStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowVnodesStmtDnodeId, &pNode->pDnodeId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkAlterDnodeStmtValue, pNode->value); + code = jsonToNodeObject(pJson, jkShowVnodesStmtDnodeEndpoint, &pNode->pDnodeEndpoint); } return code; } -static const char* jkCreateTopicStmtTopicName = "TopicName"; -static const char* jkCreateTopicStmtSubscribeDbName = "SubscribeDbName"; -static const char* jkCreateTopicStmtIgnoreExists = "IgnoreExists"; -static const char* jkCreateTopicStmtQuery = "Query"; +static int32_t showUserPrivilegesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } -static int32_t createTopicStmtToJson(const void* pObj, SJson* pJson) { - const SCreateTopicStmt* pNode = (const SCreateTopicStmt*)pObj; +static int32_t jsonToShowUserPrivilegesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } - int32_t code = tjsonAddStringToObject(pJson, jkCreateTopicStmtTopicName, pNode->topicName); +static const char* jkShowCreateDatabaseStmtDbName = "DbName"; + +static int32_t showCreateDatabaseStmtToJson(const void* pObj, SJson* pJson) { + const SShowCreateDatabaseStmt* pNode = (const SShowCreateDatabaseStmt*)pObj; + return tjsonAddStringToObject(pJson, jkShowCreateDatabaseStmtDbName, pNode->dbName); +} + +static int32_t jsonToShowCreateDatabaseStmt(const SJson* pJson, void* pObj) { + SShowCreateDatabaseStmt* pNode = (SShowCreateDatabaseStmt*)pObj; + return tjsonGetStringValue(pJson, jkShowCreateDatabaseStmtDbName, pNode->dbName); +} + +static const char* jkShowCreateTableStmtDbName = "DbName"; +static const char* jkShowCreateTableStmtTableName = "TableName"; + +static int32_t showCreateTableStmtToJson(const void* pObj, SJson* pJson) { + const SShowCreateTableStmt* pNode = (const SShowCreateTableStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkShowCreateTableStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddStringToObject(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + code = tjsonAddStringToObject(pJson, jkShowCreateTableStmtTableName, pNode->tableName); } + + return code; +} + +static int32_t jsonToShowCreateTableStmt(const SJson* pJson, void* pObj) { + SShowCreateTableStmt* pNode = (SShowCreateTableStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkShowCreateTableStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkCreateTopicStmtIgnoreExists, pNode->ignoreExists); + code = tjsonGetStringValue(pJson, jkShowCreateTableStmtTableName, pNode->tableName); + } + + return code; +} + +static int32_t showCreateStableStmtToJson(const void* pObj, SJson* pJson) { + return showCreateTableStmtToJson(pObj, pJson); +} + +static int32_t jsonToShowCreateStableStmt(const SJson* pJson, void* pObj) { + return jsonToShowCreateTableStmt(pJson, pObj); +} + +static const char* jkShowTableDistributedStmtDbName = "DbName"; +static const char* jkShowTableDistributedStmtTableName = "TableName"; + +static int32_t showTableDistributedStmtToJson(const void* pObj, SJson* pJson) { + const SShowTableDistributedStmt* pNode = (const SShowTableDistributedStmt*)pObj; + + int32_t code = tjsonAddStringToObject(pJson, jkShowTableDistributedStmtDbName, pNode->dbName); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkShowTableDistributedStmtTableName, pNode->tableName); } + + return code; +} + +static int32_t jsonToShowTableDistributedStmt(const SJson* pJson, void* pObj) { + SShowTableDistributedStmt* pNode = (SShowTableDistributedStmt*)pObj; + + int32_t code = tjsonGetStringValue(pJson, jkShowTableDistributedStmtDbName, pNode->dbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkCreateTopicStmtQuery, nodeToJson, pNode->pQuery); + code = tjsonGetStringValue(pJson, jkShowTableDistributedStmtTableName, pNode->tableName); } return code; } -static int32_t jsonToCreateTopicStmt(const SJson* pJson, void* pObj) { - SCreateTopicStmt* pNode = (SCreateTopicStmt*)pObj; +static int32_t showLocalVariablesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); } - int32_t code = tjsonGetStringValue(pJson, jkCreateTopicStmtTopicName, pNode->topicName); +static int32_t jsonToShowLocalVariablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); } + +static const char* jkShowTableTagsStmtDbName = "DbName"; +static const char* jkShowTableTagsStmtTbName = "TbName"; +static const char* jkShowTableTagsStmtTags = "Tags"; + +static int32_t showTableTagsStmtToJson(const void* pObj, SJson* pJson) { + const SShowTableTagsStmt* pNode = (const SShowTableTagsStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkShowTableTagsStmtDbName, nodeToJson, pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetStringValue(pJson, jkCreateTopicStmtSubscribeDbName, pNode->subDbName); + code = tjsonAddObject(pJson, jkShowTableTagsStmtTbName, nodeToJson, pNode->pTbName); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkCreateTopicStmtIgnoreExists, &pNode->ignoreExists); + code = nodeListToJson(pJson, jkShowTableTagsStmtTags, pNode->pTags); } + + return code; +} + +static int32_t jsonToShowTableTagsStmt(const SJson* pJson, void* pObj) { + SShowTableTagsStmt* pNode = (SShowTableTagsStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkShowTableTagsStmtDbName, &pNode->pDbName); if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkCreateTopicStmtQuery, &pNode->pQuery); + code = jsonToNodeObject(pJson, jkShowTableTagsStmtTbName, &pNode->pTbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkShowTableTagsStmtTags, &pNode->pTags); } return code; @@ -4575,6 +6252,45 @@ static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { return code; } +static const char* jkInsertStmtTable = "Table"; +static const char* jkInsertStmtCols = "Cols"; +static const char* jkInsertStmtQuery = "Query"; +static const char* jkInsertStmtPrecision = "Precision"; + +static int32_t insertStmtToJson(const void* pObj, SJson* pJson) { + const SInsertStmt* pNode = (const SInsertStmt*)pObj; + + int32_t code = tjsonAddObject(pJson, jkInsertStmtTable, nodeToJson, pNode->pTable); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkInsertStmtCols, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkInsertStmtQuery, nodeToJson, pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkInsertStmtPrecision, pNode->precision); + } + + return code; +} + +static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) { + SInsertStmt* pNode = (SInsertStmt*)pObj; + + int32_t code = jsonToNodeObject(pJson, jkInsertStmtTable, &pNode->pTable); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkInsertStmtCols, &pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkInsertStmtQuery, &pNode->pQuery); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkInsertStmtPrecision, &pNode->precision); + } + + return code; +} + static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { switch (nodeType(pObj)) { case QUERY_NODE_COLUMN: @@ -4592,7 +6308,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_TEMP_TABLE: return tempTableNodeToJson(pObj, pJson); case QUERY_NODE_JOIN_TABLE: - break; + return joinTableNodeToJson(pObj, pJson); case QUERY_NODE_GROUPING_SET: return groupingSetNodeToJson(pObj, pJson); case QUERY_NODE_ORDER_BY_EXPR: @@ -4618,42 +6334,171 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_SLOT_DESC: return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_COLUMN_DEF: - break; + return columnDefNodeToJson(pObj, pJson); case QUERY_NODE_DOWNSTREAM_SOURCE: return downstreamSourceNodeToJson(pObj, pJson); case QUERY_NODE_DATABASE_OPTIONS: return databaseOptionsToJson(pObj, pJson); + case QUERY_NODE_TABLE_OPTIONS: + return tableOptionsToJson(pObj, pJson); + case QUERY_NODE_INDEX_OPTIONS: + return indexOptionsToJson(pObj, pJson); + case QUERY_NODE_EXPLAIN_OPTIONS: + return explainOptionsToJson(pObj, pJson); + case QUERY_NODE_STREAM_OPTIONS: + return streamOptionsToJson(pObj, pJson); case QUERY_NODE_LEFT_VALUE: return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to serialize. case QUERY_NODE_WHEN_THEN: return whenThenNodeToJson(pObj, pJson); case QUERY_NODE_CASE_WHEN: return caseWhenNodeToJson(pObj, pJson); + case QUERY_NODE_EVENT_WINDOW: + return eventWindowNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: return setOperatorToJson(pObj, pJson); case QUERY_NODE_SELECT_STMT: return selectStmtToJson(pObj, pJson); - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: + return vnodeModifyStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_DATABASE_STMT: - break; + return createDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_DATABASE_STMT: return alterDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_TRIM_DATABASE_STMT: + return trimDatabaseStmtToJson(pObj, pJson); case QUERY_NODE_CREATE_TABLE_STMT: - break; + return createTableStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return createSubTableClauseToJson(pObj, pJson); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return createMultiTablesStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_TABLE_CLAUSE: + return dropTableClauseToJson(pObj, pJson); + case QUERY_NODE_DROP_TABLE_STMT: + return dropTableStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return dropStableStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_TABLE_STMT: return alterTableStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return alterStableStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_USER_STMT: + return createUserStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_USER_STMT: + return alterUserStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_USER_STMT: + return dropUserStmtToJson(pObj, pJson); case QUERY_NODE_USE_DATABASE_STMT: - break; + return useDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_DNODE_STMT: + return createDnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_DNODE_STMT: + return dropDnodeStmtToJson(pObj, pJson); case QUERY_NODE_ALTER_DNODE_STMT: return alterDnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_INDEX_STMT: + return createIndexStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_INDEX_STMT: + return dropIndexStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_QNODE_STMT: + return createQnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_QNODE_STMT: + return dropQnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_SNODE_STMT: + return createSnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_SNODE_STMT: + return dropSnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_MNODE_STMT: + return createMnodeStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_MNODE_STMT: + return dropMnodeStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_TOPIC_STMT: + return createTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_TOPIC_STMT: + return dropTopicStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_CGROUP_STMT: + return dropConsumerGroupStmtToJson(pObj, pJson); + case QUERY_NODE_ALTER_LOCAL_STMT: + return alterLocalStmtToJson(pObj, pJson); + case QUERY_NODE_EXPLAIN_STMT: + return explainStmtToJson(pObj, pJson); + case QUERY_NODE_DESCRIBE_STMT: + return describeStmtToJson(pObj, pJson); + case QUERY_NODE_COMPACT_DATABASE_STMT: + return compactDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_CREATE_STREAM_STMT: + return createStreamStmtToJson(pObj, pJson); + case QUERY_NODE_DROP_STREAM_STMT: + return dropStreamStmtToJson(pObj, pJson); + case QUERY_NODE_BALANCE_VGROUP_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to serialize. + case QUERY_NODE_MERGE_VGROUP_STMT: + return mergeVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return redistributeVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_SPLIT_VGROUP_STMT: + return splitVgroupStmtToJson(pObj, pJson); + case QUERY_NODE_GRANT_STMT: + return grantStmtToJson(pObj, pJson); + case QUERY_NODE_REVOKE_STMT: + return revokeStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_DNODES_STMT: + return showDnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_MNODES_STMT: + return showMnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_QNODES_STMT: + return showQnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CLUSTER_STMT: + return showClusterStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_DATABASES_STMT: + return showDatabasesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return showFunctionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_INDEXES_STMT: + return showIndexesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_STABLES_STMT: + return showStablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_STREAMS_STMT: + return showStreamsStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_TABLES_STMT: + return showTablesStmtToJson(pObj, pJson); case QUERY_NODE_SHOW_TAGS_STMT: - break; - case QUERY_NODE_CREATE_TOPIC_STMT: - return createTopicStmtToJson(pObj, pJson); + return showTagsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_USERS_STMT: + return showUsersStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VGROUPS_STMT: + return showVgroupsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CONSUMERS_STMT: + return showConsumersStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VARIABLES_STMT: + return showVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return showDnodeVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return showTransactionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return showSubscriptionsStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_VNODES_STMT: + return showVnodesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return showUserPrivilegesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return showCreateDatabaseStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return showCreateTableStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return showCreateStableStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return showTableDistributedStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return showLocalVariablesStmtToJson(pObj, pJson); + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return showTableTagsStmtToJson(pObj, pJson); case QUERY_NODE_DELETE_STMT: return deleteStmtToJson(pObj, pJson); + case QUERY_NODE_INSERT_STMT: + return insertStmtToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SCAN: return logicScanNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_JOIN: @@ -4726,6 +6571,9 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return physiStateWindowNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return physiEventWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return physiPartitionNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -4769,6 +6617,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToRealTableNode(pJson, pObj); case QUERY_NODE_TEMP_TABLE: return jsonToTempTableNode(pJson, pObj); + case QUERY_NODE_JOIN_TABLE: + return jsonToJoinTableNode(pJson, pObj); case QUERY_NODE_GROUPING_SET: return jsonToGroupingSetNode(pJson, pObj); case QUERY_NODE_ORDER_BY_EXPR: @@ -4791,32 +6641,176 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToDataBlockDescNode(pJson, pObj); case QUERY_NODE_SLOT_DESC: return jsonToSlotDescNode(pJson, pObj); + case QUERY_NODE_COLUMN_DEF: + return jsonToColumnDefNode(pJson, pObj); case QUERY_NODE_DOWNSTREAM_SOURCE: return jsonToDownstreamSourceNode(pJson, pObj); case QUERY_NODE_DATABASE_OPTIONS: return jsonToDatabaseOptions(pJson, pObj); + case QUERY_NODE_TABLE_OPTIONS: + return jsonToTableOptions(pJson, pObj); + case QUERY_NODE_INDEX_OPTIONS: + return jsonToIndexOptions(pJson, pObj); + case QUERY_NODE_EXPLAIN_OPTIONS: + return jsonToExplainOptions(pJson, pObj); + case QUERY_NODE_STREAM_OPTIONS: + return jsonToStreamOptions(pJson, pObj); case QUERY_NODE_LEFT_VALUE: return TSDB_CODE_SUCCESS; // SLeftValueNode has no fields to deserialize. case QUERY_NODE_WHEN_THEN: return jsonToWhenThenNode(pJson, pObj); case QUERY_NODE_CASE_WHEN: return jsonToCaseWhenNode(pJson, pObj); + case QUERY_NODE_EVENT_WINDOW: + return jsonToEventWindowNode(pJson, pObj); case QUERY_NODE_SET_OPERATOR: return jsonToSetOperator(pJson, pObj); case QUERY_NODE_SELECT_STMT: return jsonToSelectStmt(pJson, pObj); + case QUERY_NODE_VNODE_MODIFY_STMT: + return jsonToVnodeModifyStmt(pJson, pObj); + case QUERY_NODE_CREATE_DATABASE_STMT: + return jsonToCreateDatabaseStmt(pJson, pObj); case QUERY_NODE_ALTER_DATABASE_STMT: return jsonToAlterDatabaseStmt(pJson, pObj); + case QUERY_NODE_TRIM_DATABASE_STMT: + return jsonToTrimDatabaseStmt(pJson, pObj); + case QUERY_NODE_CREATE_TABLE_STMT: + return jsonToCreateTableStmt(pJson, pObj); + case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: + return jsonToCreateSubTableClause(pJson, pObj); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return jsonToCreateMultiTablesStmt(pJson, pObj); + case QUERY_NODE_DROP_TABLE_CLAUSE: + return jsonToDropTableClause(pJson, pObj); + case QUERY_NODE_DROP_TABLE_STMT: + return jsonToDropTableStmt(pJson, pObj); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return jsonToDropStableStmt(pJson, pObj); case QUERY_NODE_ALTER_TABLE_STMT: return jsonToAlterTableStmt(pJson, pObj); + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return jsonToAlterStableStmt(pJson, pObj); + case QUERY_NODE_CREATE_USER_STMT: + return jsonToCreateUserStmt(pJson, pObj); + case QUERY_NODE_ALTER_USER_STMT: + return jsonToAlterUserStmt(pJson, pObj); + case QUERY_NODE_DROP_USER_STMT: + return jsonToDropUserStmt(pJson, pObj); + case QUERY_NODE_USE_DATABASE_STMT: + return jsonToUseDatabaseStmt(pJson, pObj); + case QUERY_NODE_CREATE_DNODE_STMT: + return jsonToCreateDnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_DNODE_STMT: + return jsonToDropDnodeStmt(pJson, pObj); case QUERY_NODE_ALTER_DNODE_STMT: return jsonToAlterDnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_INDEX_STMT: + return jsonToCreateIndexStmt(pJson, pObj); + case QUERY_NODE_DROP_INDEX_STMT: + return jsonToDropIndexStmt(pJson, pObj); + case QUERY_NODE_CREATE_QNODE_STMT: + return jsonToCreateQnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_QNODE_STMT: + return jsonToDropQnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_SNODE_STMT: + return jsonToCreateSnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_SNODE_STMT: + return jsonToDropSnodeStmt(pJson, pObj); + case QUERY_NODE_CREATE_MNODE_STMT: + return jsonToCreateMnodeStmt(pJson, pObj); + case QUERY_NODE_DROP_MNODE_STMT: + return jsonToDropMnodeStmt(pJson, pObj); case QUERY_NODE_CREATE_TOPIC_STMT: return jsonToCreateTopicStmt(pJson, pObj); + case QUERY_NODE_DROP_TOPIC_STMT: + return jsonToDropTopicStmt(pJson, pObj); + case QUERY_NODE_DROP_CGROUP_STMT: + return jsonToDropConsumerGroupStmt(pJson, pObj); + case QUERY_NODE_ALTER_LOCAL_STMT: + return jsonToAlterLocalStmt(pJson, pObj); + case QUERY_NODE_EXPLAIN_STMT: + return jsonToExplainStmt(pJson, pObj); + case QUERY_NODE_DESCRIBE_STMT: + return jsonToDescribeStmt(pJson, pObj); + case QUERY_NODE_COMPACT_DATABASE_STMT: + return jsonToCompactDatabaseStmt(pJson, pObj); + case QUERY_NODE_CREATE_STREAM_STMT: + return jsonToCreateStreamStmt(pJson, pObj); + case QUERY_NODE_DROP_STREAM_STMT: + return jsonToDropStreamStmt(pJson, pObj); + case QUERY_NODE_BALANCE_VGROUP_STMT: + return TSDB_CODE_SUCCESS; // SBalanceVgroupStmt has no fields to deserialize. + case QUERY_NODE_MERGE_VGROUP_STMT: + return jsonToMergeVgroupStmt(pJson, pObj); + case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: + return jsonToRedistributeVgroupStmt(pJson, pObj); + case QUERY_NODE_SPLIT_VGROUP_STMT: + return jsonToSplitVgroupStmt(pJson, pObj); + case QUERY_NODE_GRANT_STMT: + return jsonToGrantStmt(pJson, pObj); + case QUERY_NODE_REVOKE_STMT: + return jsonToRevokeStmt(pJson, pObj); + case QUERY_NODE_SHOW_DNODES_STMT: + return jsonToShowDnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_MNODES_STMT: + return jsonToShowMnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_QNODES_STMT: + return jsonToShowQnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_CLUSTER_STMT: + return jsonToShowClusterStmt(pJson, pObj); + case QUERY_NODE_SHOW_DATABASES_STMT: + return jsonToShowDatabasesStmt(pJson, pObj); + case QUERY_NODE_SHOW_FUNCTIONS_STMT: + return jsonToShowFunctionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_INDEXES_STMT: + return jsonToShowIndexesStmt(pJson, pObj); + case QUERY_NODE_SHOW_STABLES_STMT: + return jsonToShowStablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_STREAMS_STMT: + return jsonToShowStreamsStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLES_STMT: + return jsonToShowTablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TAGS_STMT: + return jsonToShowTagsStmt(pJson, pObj); + case QUERY_NODE_SHOW_USERS_STMT: + return jsonToShowUsersStmt(pJson, pObj); + case QUERY_NODE_SHOW_VGROUPS_STMT: + return jsonToShowVgroupsStmt(pJson, pObj); + case QUERY_NODE_SHOW_CONSUMERS_STMT: + return jsonToShowConsumersStmt(pJson, pObj); + case QUERY_NODE_SHOW_VARIABLES_STMT: + return jsonToShowVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return jsonToShowDnodeVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TRANSACTIONS_STMT: + return jsonToShowTransactionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: + return jsonToShowSubscriptionsStmt(pJson, pObj); + case QUERY_NODE_SHOW_VNODES_STMT: + return jsonToShowVnodesStmt(pJson, pObj); + case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: + return jsonToShowUserPrivilegesStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + return jsonToShowCreateDatabaseStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_TABLE_STMT: + return jsonToShowCreateTableStmt(pJson, pObj); + case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + return jsonToShowCreateStableStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: + return jsonToShowTableDistributedStmt(pJson, pObj); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return jsonToShowLocalVariablesStmt(pJson, pObj); + case QUERY_NODE_SHOW_TABLE_TAGS_STMT: + return jsonToShowTableTagsStmt(pJson, pObj); case QUERY_NODE_DELETE_STMT: return jsonToDeleteStmt(pJson, pObj); + case QUERY_NODE_INSERT_STMT: + return jsonToInsertStmt(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SCAN: return jsonToLogicScanNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_JOIN: + return jsonToLogicJoinNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_AGG: return jsonToLogicAggNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_PROJECT: @@ -4885,6 +6879,9 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return jsonToPhysiStateWindowNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return jsonToPhysiEventWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 104f066fa945572bdc8add88452d133668a39b3e..ad80508c64a776154860a4ea21b3c75524514e31 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2945,6 +2945,46 @@ static int32_t msgToPhysiStateWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } +enum { PHY_EVENT_CODE_WINDOW = 1, PHY_EVENT_CODE_START_COND, PHY_EVENT_CODE_END_COND }; + +static int32_t physiEventWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEventWinodwPhysiNode* pNode = (const SEventWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_START_COND, nodeToMsg, pNode->pStartCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EVENT_CODE_END_COND, nodeToMsg, pNode->pEndCond); + } + + return code; +} + +static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) { + SEventWinodwPhysiNode* pNode = (SEventWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_EVENT_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_EVENT_CODE_START_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartCond); + break; + case PHY_EVENT_CODE_END_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndCond); + break; + default: + break; + } + } + + return code; +} + enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -3215,7 +3255,8 @@ enum { PHY_QUERY_INSERT_CODE_TABLE_TYPE, PHY_QUERY_INSERT_CODE_TABLE_NAME, PHY_QUERY_INSERT_CODE_VG_ID, - PHY_QUERY_INSERT_CODE_EP_SET + PHY_QUERY_INSERT_CODE_EP_SET, + PHY_QUERY_INSERT_CODE_EXPLAIN }; static int32_t physiQueryInsertNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -3243,6 +3284,9 @@ static int32_t physiQueryInsertNodeToMsg(const void* pObj, STlvEncoder* pEncoder if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_EP_SET, epSetToMsg, &pNode->epSet); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_QUERY_INSERT_CODE_EXPLAIN, pNode->explain); + } return code; } @@ -3278,6 +3322,9 @@ static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { case PHY_QUERY_INSERT_CODE_EP_SET: code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->epSet); break; + case PHY_QUERY_INSERT_CODE_EXPLAIN: + code = tlvDecodeBool(pTlv, &pNode->explain); + break; default: break; } @@ -3716,6 +3763,10 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = physiStateWindowNodeToMsg(pObj, pEncoder); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = physiEventWindowNodeToMsg(pObj, pEncoder); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = physiPartitionNodeToMsg(pObj, pEncoder); break; @@ -3855,6 +3906,10 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: code = msgToPhysiStateWindowNode(pDecoder, pObj); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + code = msgToPhysiEventWindowNode(pDecoder, pObj); + break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: code = msgToPhysiPartitionNode(pDecoder, pObj); break; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 106812d55f1cdd08120cb100e338b0b82244e752..ce575ede8a5dd2020c6af39394a72d6fa39cc348 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -165,6 +165,17 @@ static EDealRes dispatchExpr(SNode* pNode, ETraversalOrder order, FNodeWalker wa } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = walkExpr(pEvent->pCol, order, walker, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pStartCond, order, walker, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = walkExpr(pEvent->pEndCond, order, walker, pContext); + } + break; + } default: break; } @@ -329,6 +340,17 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit } break; } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext); + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pStartCond, order, rewriter, pContext); + } + if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { + res = rewriteExpr(&pEvent->pEndCond, order, rewriter, pContext); + } + break; + } default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index e28486e0b63c9c44e9ea766b5c497b836ba4fdf1..0419c883e6320aea75743f9106a65606668e21a4 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -299,12 +299,14 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SWhenThenNode)); case QUERY_NODE_CASE_WHEN: return makeNode(type, sizeof(SCaseWhenNode)); + case QUERY_NODE_EVENT_WINDOW: + return makeNode(type, sizeof(SEventWindowNode)); case QUERY_NODE_SET_OPERATOR: return makeNode(type, sizeof(SSetOperator)); case QUERY_NODE_SELECT_STMT: return makeNode(type, sizeof(SSelectStmt)); - case QUERY_NODE_VNODE_MODIF_STMT: - return makeNode(type, sizeof(SVnodeModifOpStmt)); + case QUERY_NODE_VNODE_MODIFY_STMT: + return makeNode(type, sizeof(SVnodeModifyOpStmt)); case QUERY_NODE_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SCreateDatabaseStmt)); case QUERY_NODE_DROP_DATABASE_STMT: @@ -319,8 +321,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SCreateTableStmt)); case QUERY_NODE_CREATE_SUBTABLE_CLAUSE: return makeNode(type, sizeof(SCreateSubTableClause)); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return makeNode(type, sizeof(SCreateMultiTableStmt)); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return makeNode(type, sizeof(SCreateMultiTablesStmt)); case QUERY_NODE_DROP_TABLE_CLAUSE: return makeNode(type, sizeof(SDropTableClause)); case QUERY_NODE_DROP_TABLE_STMT: @@ -372,8 +374,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SDescribeStmt)); case QUERY_NODE_RESET_QUERY_CACHE_STMT: return makeNode(type, sizeof(SNode)); - case QUERY_NODE_COMPACT_STMT: - break; + case QUERY_NODE_COMPACT_DATABASE_STMT: + return makeNode(type, sizeof(SCompactDatabaseStmt)); case QUERY_NODE_CREATE_FUNCTION_STMT: return makeNode(type, sizeof(SCreateFunctionStmt)); case QUERY_NODE_DROP_FUNCTION_STMT: @@ -432,6 +434,9 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SShowDnodeVariablesStmt)); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SShowCreateDatabaseStmt)); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return makeNode(type, sizeof(SShowAliveStmt)); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return makeNode(type, sizeof(SShowCreateTableStmt)); @@ -535,6 +540,10 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + return makeNode(type, sizeof(SEventWinodwPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + return makeNode(type, sizeof(SStreamEventWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: @@ -765,16 +774,23 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_COLUMN_REF: // no pointer field break; case QUERY_NODE_WHEN_THEN: { - SWhenThenNode* pStmt = (SWhenThenNode*)pNode; - nodesDestroyNode(pStmt->pWhen); - nodesDestroyNode(pStmt->pThen); + SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + nodesDestroyNode(pWhenThen->pWhen); + nodesDestroyNode(pWhenThen->pThen); break; } case QUERY_NODE_CASE_WHEN: { - SCaseWhenNode* pStmt = (SCaseWhenNode*)pNode; - nodesDestroyNode(pStmt->pCase); - nodesDestroyNode(pStmt->pElse); - nodesDestroyList(pStmt->pWhenThenList); + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; + nodesDestroyNode(pCaseWhen->pCase); + nodesDestroyNode(pCaseWhen->pElse); + nodesDestroyList(pCaseWhen->pWhenThenList); + break; + } + case QUERY_NODE_EVENT_WINDOW: { + SEventWindowNode* pEvent = (SEventWindowNode*)pNode; + nodesDestroyNode(pEvent->pCol); + nodesDestroyNode(pEvent->pStartCond); + nodesDestroyNode(pEvent->pEndCond); break; } case QUERY_NODE_SET_OPERATOR: { @@ -805,8 +821,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pSlimit); break; } - case QUERY_NODE_VNODE_MODIF_STMT: { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pNode; + case QUERY_NODE_VNODE_MODIFY_STMT: { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; destroyVgDataBlockArray(pStmt->pDataBlocks); taosMemoryFreeClear(pStmt->pTableMeta); taosHashCleanup(pStmt->pVgroupsHashObj); @@ -819,7 +835,8 @@ void nodesDestroyNode(SNode* pNode) { if (pStmt->freeArrayFunc) { pStmt->freeArrayFunc(pStmt->pVgDataBlocks); } - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); taosCloseFile(&pStmt->fp); break; } @@ -848,8 +865,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode((SNode*)pStmt->pOptions); break; } - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - nodesDestroyList(((SCreateMultiTableStmt*)pNode)->pSubTables); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + nodesDestroyList(((SCreateMultiTablesStmt*)pNode)->pSubTables); break; case QUERY_NODE_DROP_TABLE_CLAUSE: // no pointer field break; @@ -906,7 +923,7 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFree(((SDescribeStmt*)pNode)->pMeta); break; case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field - case QUERY_NODE_COMPACT_STMT: // no pointer field + case QUERY_NODE_COMPACT_DATABASE_STMT: // no pointer field case QUERY_NODE_CREATE_FUNCTION_STMT: // no pointer field case QUERY_NODE_DROP_FUNCTION_STMT: // no pointer field break; @@ -946,6 +963,8 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_USERS_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_CONSUMERS_STMT: case QUERY_NODE_SHOW_CONNECTIONS_STMT: @@ -1084,6 +1103,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pLogicNode->pTspk); nodesDestroyNode(pLogicNode->pTsEnd); nodesDestroyNode(pLogicNode->pStateExpr); + nodesDestroyNode(pLogicNode->pStartCond); + nodesDestroyNode(pLogicNode->pEndCond); break; } case QUERY_NODE_LOGIC_PLAN_FILL: { @@ -1232,6 +1253,14 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pStateKey); break; } + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: { + SEventWinodwPhysiNode* pPhyNode = (SEventWinodwPhysiNode*)pNode; + destroyWinodwPhysiNode((SWinodwPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pStartCond); + nodesDestroyNode(pPhyNode->pEndCond); + break; + } case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { destroyPartitionPhysiNode((SPartitionPhysiNode*)pNode); break; diff --git a/source/libs/nodes/test/nodesTestMain.cpp b/source/libs/nodes/test/nodesTestMain.cpp index 0515e8bbc0ae1319042f868b4d70315cc1e31b9c..a7f9a06611b3bd6bec2283b483127ab4737d03b9 100644 --- a/source/libs/nodes/test/nodesTestMain.cpp +++ b/source/libs/nodes/test/nodesTestMain.cpp @@ -28,7 +28,7 @@ static EDealRes rewriterTest(SNode** pNode, void* pContext) { } SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); string tmp = to_string(stoi(((SValueNode*)(pOp->pLeft))->literal) + stoi(((SValueNode*)(pOp->pRight))->literal)); - pVal->literal = strdup(tmp.c_str()); + pVal->literal = taosStrdup(tmp.c_str()); nodesDestroyNode(*pNode); *pNode = (SNode*)pVal; } @@ -40,12 +40,12 @@ TEST(NodesTest, traverseTest) { SOperatorNode* pOp = (SOperatorNode*)pRoot; SOperatorNode* pLeft = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); pLeft->pLeft = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); - ((SValueNode*)(pLeft->pLeft))->literal = strdup("10"); + ((SValueNode*)(pLeft->pLeft))->literal = taosStrdup("10"); pLeft->pRight = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); - ((SValueNode*)(pLeft->pRight))->literal = strdup("5"); + ((SValueNode*)(pLeft->pRight))->literal = taosStrdup("5"); pOp->pLeft = (SNode*)pLeft; pOp->pRight = (SNode*)nodesMakeNode(QUERY_NODE_VALUE); - ((SValueNode*)(pOp->pRight))->literal = strdup("3"); + ((SValueNode*)(pOp->pRight))->literal = taosStrdup("3"); EXPECT_EQ(nodeType(pRoot), QUERY_NODE_OPERATOR); EDealRes res = DEAL_RES_CONTINUE; diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 5fecc95f1bbe7f21091803b2fe6d705eda3efd4b..3b309db1db61b1d1e99918962fe73bd1ffa5cb09 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -116,6 +116,7 @@ SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const STok SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr); +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond); SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); @@ -150,6 +151,7 @@ SNode* createDropDatabaseStmt(SAstCreateContext* pCxt, bool ignoreNotExists, STo SNode* createAlterDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, SNode* pOptions); SNode* createFlushDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t maxSpeed); +SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createDefaultTableOptions(SAstCreateContext* pCxt); SNode* createAlterTableOptions(SAstCreateContext* pCxt); SNode* setTableOption(SAstCreateContext* pCxt, SNode* pOptions, ETableOptionType type, void* pVal); @@ -175,6 +177,7 @@ SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type); SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pDbName, SNode* pTbName, EOperatorType tableCondType); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern); @@ -208,13 +211,12 @@ SNode* setExplainRatio(SAstCreateContext* pCxt, SNode* pOptions, const SToken* p SNode* createExplainStmt(SAstCreateContext* pCxt, bool analyze, SNode* pOptions, SNode* pQuery); SNode* createDescribeStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt); -SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups); SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize); SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName); SNode* createStreamOptions(SAstCreateContext* pCxt); SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery); + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols); SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName); SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId); SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId); diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index 9dd4daec3b3905740aa8ee2d25bcbd7751279aec..303d349b34a1d182bd65a839f866a5b70892afad 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -20,8 +20,6 @@ struct SToken; -#define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED) - #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -37,108 +35,27 @@ struct SToken; } \ } while (0) -typedef enum EOrderStatus { - ORDER_STATUS_UNKNOWN = 0, - ORDER_STATUS_ORDERED = 1, - ORDER_STATUS_DISORDERED = 2, -} EOrderStatus; - -typedef enum EValStat { - VAL_STAT_HAS = 0x0, // 0 means has val - VAL_STAT_NONE = 0x01, // 1 means no val -} EValStat; - -typedef struct SBoundColumn { - int32_t offset; // all column offset value - int32_t toffset; // first part offset for SDataRow TODO: get offset from STSchema on future - uint8_t valStat; // EValStat. denote if current column bound or not(0 means has val, 1 means no val) -} SBoundColumn; - -typedef struct { - col_id_t schemaColIdx; - col_id_t boundIdx; - col_id_t finalIdx; -} SBoundIdxInfo; - -typedef struct SParsedDataColInfo { - col_id_t numOfCols; - col_id_t numOfBound; - uint16_t flen; // TODO: get from STSchema - uint16_t allNullLen; // TODO: get from STSchema(base on SDataRow) - uint16_t extendedVarLen; - uint16_t boundNullLen; // bound column len with all NULL value(without VarDataOffsetT/SColIdx part) - col_id_t *boundColumns; // bound column idx according to schema - SBoundColumn *cols; - SBoundIdxInfo *colIdxInfo; - int8_t orderStatus; // bound columns -} SParsedDataColInfo; - -typedef struct SInsertParseBaseContext { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; -} SInsertParseBaseContext; - -typedef struct SInsertParseSyntaxCxt { - SParseContext *pComCxt; - char *pSql; - SMsgBuf msg; - SParseMetaCache *pMetaCache; -} SInsertParseSyntaxCxt; - -typedef struct SMemParam { - SRowBuilder *rb; - SSchema *schema; - int32_t toffset; - col_id_t colIdx; -} SMemParam; - -typedef struct { - uint8_t rowType; // default is 0, that is SDataRow - int32_t rowSize; -} SMemRowBuilder; - -typedef struct STableDataBlocks { - int8_t tsSource; // where does the UNIX timestamp come from, server or client - bool ordered; // if current rows are ordered or not - int32_t vgId; // virtual group id - int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending - int32_t numOfTables; // number of tables in current submit block - int32_t rowSize; // row size for current table - uint32_t nAllocSize; - uint32_t headerSize; // header for table info (uid, tid, submit metadata) - uint32_t size; - STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to - // avoid to be removed from cache - char *pData; - bool cloned; - int32_t createTbReqLen; - SParsedDataColInfo boundColumnInfo; - SRowBuilder rowBuilder; -} STableDataBlocks; +typedef struct SVgroupDataCxt { + int32_t vgId; + SSubmitReq2 *pData; +} SVgroupDataCxt; -int32_t insGetExtendedRowSize(STableDataBlocks *pBlock); -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo *spd, col_id_t idx, int32_t *toffset, col_id_t *colIdx); -int32_t insSetBlockInfo(SSubmitBlk *pBlocks, STableDataBlocks *dataBuf, int32_t numOfRows, SMsgBuf *pMsg); -int32_t insSchemaIdxCompar(const void *lhs, const void *rhs); -int32_t insBoundIdxCompar(const void *lhs, const void *rhs); -void insSetBoundColumnInfo(SParsedDataColInfo *pColList, SSchema *pSchema, col_id_t numOfCols); -void insDestroyBlockArrayList(SArray *pDataBlockList); -void insDestroyBlockHashmap(SHashObj *pDataBlockHash); -int32_t insInitRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo); -int32_t insGetDataBlockFromList(SHashObj *pHashList, void *id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta *pTableMeta, STableDataBlocks **dataBlocks, - SArray *pBlockList, SVCreateTbReq *pCreateTbReq); -int32_t insMergeTableDataBlocks(SHashObj *pHashObj, SArray **pVgDataBlocks); -int32_t insBuildCreateTbMsg(STableDataBlocks *pBlocks, SVCreateTbReq *pCreateTbReq); -int32_t insAllocateMemForSize(STableDataBlocks *pDataBlock, int32_t allSize); int32_t insCreateSName(SName *pName, struct SToken *pTableName, int32_t acctId, const char *dbName, SMsgBuf *pMsgBuf); -int32_t insFindCol(struct SToken *pColname, int32_t start, int32_t end, SSchema *pSchema); +int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema *pSchema); void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); -int32_t insMemRowAppend(SMsgBuf *pMsgBuf, const void *value, int32_t len, void *param); -int32_t insCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start); -int32_t insBuildOutput(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); -void insDestroyDataBlock(STableDataBlocks *pDataBlock); +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); +int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode); +int32_t initTableColSubmitData(STableDataCxt *pTableCxt); +int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks); +int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); +void insDestroyTableDataCxtHashMap(SHashObj *pTableCxtHash); +void insDestroyVgroupDataCxt(SVgroupDataCxt *pVgCxt); +void insDestroyVgroupDataCxtList(SArray *pVgCxtList); +void insDestroyVgroupDataCxtHashMap(SHashObj *pVgCxtHash); +void insDestroyTableDataCxt(STableDataCxt *pTableCxt); +void insDestroyBoundColInfo(SBoundColInfo *pInfo); #endif // TDENGINE_PAR_INSERT_UTIL_H diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index d539e8b37bfbab8cd0995f2f77c8228809b07f7e..86bcc18fd5ed91a39c285a6c98357cf04154352c 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -175,8 +175,6 @@ _end: void taosCleanupKeywordsTable(); -SToken taosTokenDup(SToken *pToken, char *buf, int32_t len); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index faa908021e0123998362d6e19ded991cf97ead5a..fb6d3bfdd93bcc3ef14ba998ee9985e46c334dac 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -167,6 +167,7 @@ cmd ::= USE db_name(A). cmd ::= ALTER DATABASE db_name(A) alter_db_options(B). { pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &A, B); } cmd ::= FLUSH DATABASE db_name(A). { pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &A); } cmd ::= TRIM DATABASE db_name(A) speed_opt(B). { pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &A, B); } +cmd ::= COMPACT DATABASE db_name(A). { pCxt->pRootNode = createCompactStmt(pCxt, &A); } %type not_exists_opt { bool } %destructor not_exists_opt { } @@ -300,7 +301,7 @@ create_subtable_clause(A) ::= %type multi_drop_clause { SNodeList* } %destructor multi_drop_clause { nodesDestroyList($$); } multi_drop_clause(A) ::= drop_table_clause(B). { A = createNodeList(pCxt, B); } -multi_drop_clause(A) ::= multi_drop_clause(B) drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } +multi_drop_clause(A) ::= multi_drop_clause(B) NK_COMMA drop_table_clause(C). { A = addNodeToList(pCxt, B, C); } drop_table_clause(A) ::= exists_opt(B) full_table_name(C). { A = createDropTableClause(pCxt, B, C); } @@ -433,6 +434,9 @@ cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). cmd ::= SHOW TABLE TAGS tag_list_opt(C) FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowTableTagsStmt(pCxt, A, B, C); } cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } +// show alive +cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); } +cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createIdentifierValueNode(pCxt, &B); } @@ -458,17 +462,19 @@ tag_item(A) ::= column_name(B) column_alias(C). tag_item(A) ::= column_name(B) AS column_alias(C). { A = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &B), &C); } /************************************************ create index ********************************************************/ -cmd ::= CREATE SMA INDEX not_exists_opt(D) +cmd ::= CREATE SMA INDEX not_exists_opt(D) full_index_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); } +cmd ::= CREATE INDEX not_exists_opt(D) + full_index_name(A) ON full_table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, D, A, B, C, NULL); } cmd ::= DROP INDEX exists_opt(B) full_index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); } full_index_name(A) ::= index_name(B). { A = createRealTableNodeForIndexName(pCxt, NULL, &B); } full_index_name(A) ::= db_name(B) NK_DOT index_name(C). { A = createRealTableNodeForIndexName(pCxt, &B, &C); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); } -index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL - NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) +index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL + NK_LP duration_literal(C) NK_COMMA duration_literal(D) NK_RP sliding_opt(E) sma_stream_opt(F). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D), E, F); } %type func_list { SNodeList* } @@ -476,7 +482,15 @@ index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL func_list(A) ::= func(B). { A = createNodeList(pCxt, B); } func_list(A) ::= func_list(B) NK_COMMA func(C). { A = addNodeToList(pCxt, B, C); } -func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } +func(A) ::= sma_func_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } + +%type sma_func_name { SToken } +%destructor sma_func_name { } +sma_func_name(A) ::= function_name(B). { A = B; } +sma_func_name(A) ::= COUNT(B). { A = B; } +sma_func_name(A) ::= FIRST(B). { A = B; } +sma_func_name(A) ::= LAST(B). { A = B; } +sma_func_name(A) ::= LAST_ROW(B). { A = B; } sma_stream_opt(A) ::= . { A = createStreamOptions(pCxt); } sma_stream_opt(A) ::= sma_stream_opt(B) WATERMARK duration_literal(C). { ((SStreamOptions*)B)->pWatermark = releaseRawExprNode(pCxt, C); A = B; } @@ -504,6 +518,7 @@ cmd ::= RESET QUERY CACHE. /************************************************ explain *************************************************************/ cmd ::= EXPLAIN analyze_opt(A) explain_options(B) query_or_subquery(C). { pCxt->pRootNode = createExplainStmt(pCxt, A, B, C); } +cmd ::= EXPLAIN analyze_opt(A) explain_options(B) insert_query(C). { pCxt->pRootNode = createExplainStmt(pCxt, A, B, C); } %type analyze_opt { bool } %destructor analyze_opt { } @@ -514,9 +529,6 @@ explain_options(A) ::= . explain_options(A) ::= explain_options(B) VERBOSE NK_BOOL(C). { A = setExplainVerbose(pCxt, B, &C); } explain_options(A) ::= explain_options(B) RATIO NK_FLOAT(C). { A = setExplainRatio(pCxt, B, &C); } -/************************************************ compact *************************************************************/ -//cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP. { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - /************************************************ create/drop function ************************************************/ cmd ::= CREATE agg_func_opt(A) FUNCTION not_exists_opt(F) function_name(B) AS NK_STRING(C) OUTPUTTYPE type_name(D) bufsize_opt(E). { pCxt->pRootNode = createCreateFunctionStmt(pCxt, F, A, &B, &C, D, E); } @@ -534,9 +546,21 @@ bufsize_opt(A) ::= BUFSIZE NK_INTEGER(B). /************************************************ create/drop stream **************************************************/ cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO - full_table_name(C) tags_def_opt(F) subtable_opt(G) AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D); } + full_table_name(C) col_list_opt(H) tag_def_or_ref_opt(F) subtable_opt(G) + AS query_or_subquery(D). { pCxt->pRootNode = createCreateStreamStmt(pCxt, E, &A, C, B, F, G, D, H); } cmd ::= DROP STREAM exists_opt(A) stream_name(B). { pCxt->pRootNode = createDropStreamStmt(pCxt, A, &B); } +%type col_list_opt { SNodeList* } +%destructor col_list_opt { nodesDestroyList($$); } +col_list_opt(A) ::= . { A = NULL; } +col_list_opt(A) ::= NK_LP col_name_list(B) NK_RP. { A = B; } + +%type tag_def_or_ref_opt { SNodeList* } +%destructor tag_def_or_ref_opt { nodesDestroyList($$); } +tag_def_or_ref_opt(A) ::= . { A = NULL; } +tag_def_or_ref_opt(A) ::= tags_def(B). { A = B; } +tag_def_or_ref_opt(A) ::= TAGS NK_LP col_name_list(B) NK_RP. { A = B; } + stream_options(A) ::= . { A = createStreamOptions(pCxt); } stream_options(A) ::= stream_options(B) TRIGGER AT_ONCE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_AT_ONCE; A = B; } stream_options(A) ::= stream_options(B) TRIGGER WINDOW_CLOSE. { ((SStreamOptions*)B)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; A = B; } @@ -576,9 +600,11 @@ cmd ::= DELETE FROM full_table_name(A) where_clause_opt(B). cmd ::= query_or_subquery(A). { pCxt->pRootNode = A; } /************************************************ insert **************************************************************/ -cmd ::= INSERT INTO full_table_name(A) - NK_LP col_name_list(B) NK_RP query_or_subquery(C). { pCxt->pRootNode = createInsertStmt(pCxt, A, B, C); } -cmd ::= INSERT INTO full_table_name(A) query_or_subquery(B). { pCxt->pRootNode = createInsertStmt(pCxt, A, NULL, B); } +cmd ::= insert_query(A). { pCxt->pRootNode = A; } + +insert_query(A) ::= INSERT INTO full_table_name(D) + NK_LP col_name_list(B) NK_RP query_or_subquery(C). { A = createInsertStmt(pCxt, D, B, C); } +insert_query(A) ::= INSERT INTO full_table_name(C) query_or_subquery(B). { A = createInsertStmt(pCxt, C, NULL, B); } /************************************************ literal *************************************************************/ literal(A) ::= NK_INTEGER(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &B)); } @@ -742,6 +768,7 @@ pseudo_column(A) ::= WSTART(B). pseudo_column(A) ::= WEND(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= WDURATION(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= IROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } +pseudo_column(A) ::= ISFILLED(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } pseudo_column(A) ::= QTAGS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } @@ -973,6 +1000,8 @@ twindow_clause_opt(A) ::= twindow_clause_opt(A) ::= INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP sliding_opt(D) fill_opt(E). { A = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), D, E); } +twindow_clause_opt(A) ::= + EVENT_WINDOW START WITH search_condition(B) END WITH search_condition(C). { A = createEventWindowNode(pCxt, B, C); } sliding_opt(A) ::= . { A = NULL; } sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { A = releaseRawExprNode(pCxt, B); } @@ -1077,6 +1106,6 @@ null_ordering_opt(A) ::= . null_ordering_opt(A) ::= NULLS FIRST. { A = NULL_ORDER_FIRST; } null_ordering_opt(A) ::= NULLS LAST. { A = NULL_ORDER_LAST; } -%fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL +%fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY MODULES NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT STRICT STRING TIMES VALUES VARIABLE VIEW WAL. diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index d1f861a5fc160fa06af8a57b18e26402c91cbecf..f613b1bd3d78d7ab1b8660a5e642896ffcbb9f43 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -355,7 +355,7 @@ SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt) { SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); CHECK_OUT_OF_MEM(val); - val->literal = strdup(pCxt->pQueryCxt->db); + val->literal = taosStrdup(pCxt->pQueryCxt->db); CHECK_OUT_OF_MEM(val->literal); val->isDuration = false; val->translate = false; @@ -623,6 +623,20 @@ SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr) { return (SNode*)state; } +SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond) { + CHECK_PARSER_STATUS(pCxt); + SEventWindowNode* pEvent = (SEventWindowNode*)nodesMakeNode(QUERY_NODE_EVENT_WINDOW); + CHECK_OUT_OF_MEM(pEvent); + pEvent->pCol = createPrimaryKeyCol(pCxt, NULL); + if (NULL == pEvent->pCol) { + nodesDestroyNode((SNode*)pEvent); + CHECK_OUT_OF_MEM(NULL); + } + pEvent->pStartCond = pStartCond; + pEvent->pEndCond = pEndCond; + return (SNode*)pEvent; +} + SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { CHECK_PARSER_STATUS(pCxt); @@ -1091,6 +1105,17 @@ SNode* createTrimDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName, int32_t return (SNode*)pStmt; } +SNode* createCompactStmt(SAstCreateContext* pCxt, SToken* pDbName) { + CHECK_PARSER_STATUS(pCxt); + if (!checkDbName(pCxt, pDbName, false)) { + return NULL; + } + SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)nodesMakeNode(QUERY_NODE_COMPACT_DATABASE_STMT); + CHECK_OUT_OF_MEM(pStmt); + COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbName); + return (SNode*)pStmt; +} + SNode* createDefaultTableOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); STableOptions* pOptions = (STableOptions*)nodesMakeNode(QUERY_NODE_TABLE_OPTIONS); @@ -1213,7 +1238,7 @@ SNode* createCreateSubTableClause(SAstCreateContext* pCxt, bool ignoreExists, SN SNode* createCreateMultiTableStmt(SAstCreateContext* pCxt, SNodeList* pSubTables) { CHECK_PARSER_STATUS(pCxt); - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLE_STMT); + SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)nodesMakeNode(QUERY_NODE_CREATE_MULTI_TABLES_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->pSubTables = pSubTables; return (SNode*)pStmt; @@ -1374,6 +1399,38 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) { return (SNode*)pStmt; } +SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pNode, ENodeType type) { + CHECK_PARSER_STATUS(pCxt); + SToken dbToken = {0}; + SToken* pDbToken = NULL; + + if (pNode) { + SValueNode* pDbName = (SValueNode*)pNode; + if (pDbName->literal) { + dbToken.z = pDbName->literal; + dbToken.n = strlen(pDbName->literal); + pDbToken = &dbToken; + } + } + + if (pDbToken && !checkDbName(pCxt, pDbToken, true)) { + nodesDestroyNode(pNode); + return NULL; + } + + SShowAliveStmt* pStmt = (SShowAliveStmt*)nodesMakeNode(type); + CHECK_OUT_OF_MEM(pStmt); + + if (pDbToken) { + COPY_STRING_FORM_ID_TOKEN(pStmt->dbName, pDbToken); + } + if (pNode) { + nodesDestroyNode(pNode); + } + + return (SNode*)pStmt; +} + SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) { CHECK_PARSER_STATUS(pCxt); SShowCreateTableStmt* pStmt = (SShowCreateTableStmt*)nodesMakeNode(type); @@ -1435,7 +1492,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST } SCreateUserStmt* pStmt = (SCreateUserStmt*)nodesMakeNode(QUERY_NODE_CREATE_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); strcpy(pStmt->password, password); pStmt->sysinfo = sysinfo; return (SNode*)pStmt; @@ -1448,7 +1505,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al } SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); pStmt->alterType = alterType; switch (alterType) { case TSDB_ALTER_USER_PASSWD: { @@ -1479,7 +1536,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName) { } SDropUserStmt* pStmt = (SDropUserStmt*)nodesMakeNode(QUERY_NODE_DROP_USER_STMT); CHECK_OUT_OF_MEM(pStmt); - COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); + COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); return (SNode*)pStmt; } @@ -1722,13 +1779,6 @@ SNode* createResetQueryCacheStmt(SAstCreateContext* pCxt) { return pStmt; } -SNode* createCompactStmt(SAstCreateContext* pCxt, SNodeList* pVgroups) { - CHECK_PARSER_STATUS(pCxt); - SNode* pStmt = nodesMakeNode(QUERY_NODE_COMPACT_STMT); - CHECK_OUT_OF_MEM(pStmt); - return pStmt; -} - SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool aggFunc, const SToken* pFuncName, const SToken* pLibPath, SDataType dataType, int32_t bufSize) { CHECK_PARSER_STATUS(pCxt); @@ -1768,7 +1818,7 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) { } SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable, - SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) { + SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery, SNodeList* pCols) { CHECK_PARSER_STATUS(pCxt); if (!checkStreamName(pCxt, pStreamName)) { return NULL; @@ -1776,16 +1826,15 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName); - if (NULL != pRealTable) { - strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); - nodesDestroyNode(pRealTable); - } + strcpy(pStmt->targetDbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->targetTabName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); pStmt->ignoreExists = ignoreExists; pStmt->pOptions = (SStreamOptions*)pOptions; pStmt->pQuery = pQuery; pStmt->pTags = pTags; pStmt->pSubtable = pSubtable; + pStmt->pCols = pCols; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 8346ab8c05e7dd606b0223ad53663d69386d7927..126027c78f0629cf5490cce490cc819fa4e2984f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -166,7 +166,7 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code && - (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) && + (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS) || 0 == strcmp(pTable, TSDB_INS_TABLE_TABLES) || 0 == strcmp(pTable, TSDB_INS_TABLE_COLS)) && QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { code = collectMetaKeyFromInsTags(pCxt); } @@ -252,7 +252,7 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa return code; } -static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTableStmt* pStmt) { +static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { @@ -355,7 +355,12 @@ static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStm } static int32_t collectMetaKeyFromCreateStream(SCollectMetaKeyCxt* pCxt, SCreateStreamStmt* pStmt) { - return collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + int32_t code = + reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->targetDbName, pStmt->targetTabName, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromQuery(pCxt, pStmt->pQuery); + } + return code; } static int32_t collectMetaKeyFromShowDnodes(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { @@ -613,8 +618,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromFlushDatabase(pCxt, (SFlushDatabaseStmt*)pStmt); case QUERY_NODE_CREATE_TABLE_STMT: return collectMetaKeyFromCreateTable(pCxt, (SCreateTableStmt*)pStmt); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_DROP_TABLE_STMT: return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); case QUERY_NODE_ALTER_TABLE_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 4fd711e0ea43a0f82b09d598dca866d8ef682b04..e4de60fd051d6f7f4e781160305ffb95a3ec7d3e 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -78,7 +78,7 @@ static int32_t authSetOperator(SAuthCxt* pCxt, SSetOperator* pSetOper) { } static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { - if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->useName, TSDB_DEFAULT_USER)) { + if (!pCxt->pParseCxt->isSuperUser || 0 == strcmp(pStmt->userName, TSDB_DEFAULT_USER)) { return TSDB_CODE_PAR_PERMISSION_DENIED; } return TSDB_CODE_SUCCESS; @@ -108,7 +108,7 @@ static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE); } -static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTableStmt* pStmt) { +static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { @@ -134,8 +134,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { return authInsert(pCxt, (SInsertStmt*)pStmt); case QUERY_NODE_CREATE_TABLE_STMT: return authCreateTable(pCxt, (SCreateTableStmt*)pStmt); - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: - return authCreateMultiTable(pCxt, (SCreateMultiTableStmt*)pStmt); + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: + return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MODULES_STMT: @@ -145,6 +145,8 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 358baa74cb3932887894bda79a6e26e9a888c797..4742921d083257b56d7095d0e8aca8fe705900c2 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -18,6 +18,16 @@ #include "parToken.h" #include "ttime.h" +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (TSDB_DATA_TYPE_NCHAR == pCol->type) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf, int32_t msgBufLen) { SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen}; @@ -45,95 +55,46 @@ int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* return TSDB_CODE_SUCCESS; } -typedef struct SmlExecTableHandle { - SParsedDataColInfo tags; // each table - SVCreateTbReq createTblReq; // each table -} SmlExecTableHandle; - -typedef struct SmlExecHandle { - SHashObj* pBlockHash; - SmlExecTableHandle tableExecHandle; - SQuery* pQuery; -} SSmlExecHandle; - -static void smlDestroyTableHandle(void* pHandle) { - SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; - destroyBoundColumnInfo(&handle->tags); - tdDestroySVCreateTbReq(&handle->createTblReq); -} - -static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SSchema* pSchema, bool isTag) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found + pBoundInfo->numOfBound = 0; + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < taosArrayGetSize(cols); ++i) { - SSmlKv* kv = taosArrayGetP(cols, i); + SSmlKv* kv = taosArrayGet(cols, i); SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; - col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, nCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols); + col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema)); + uDebug("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols); if (index < 0 && t > 0) { index = insFindCol(&sToken, 0, t, pSchema); - isOrdered = false; } + if (index < 0) { uError("smlBoundColumnData. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { + if (pUseCols[index]) { uError("smlBoundColumnData. already set. index:%d", index); - return TSDB_CODE_SML_INVALID_DATA; + code = TSDB_CODE_SML_INVALID_DATA; + goto end; } lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; - } + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } +end: + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } /** @@ -146,7 +107,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS * @param msg * @return int32_t */ -static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, +static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName, SMsgBuf* msg) { SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); if (!pTagArray) { @@ -159,8 +120,8 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { - SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - SSmlKv* kv = taosArrayGetP(cols, i); + SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; + SSmlKv* kv = taosArrayGet(cols, i); taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; @@ -207,153 +168,253 @@ end: return code; } -int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, - char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int16_t msgBufLen) { - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { + STableDataCxt* pTableCxt = NULL; + SVCreateTbReq* pCreateTbReq = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false); + if (ret != TSDB_CODE_SUCCESS) { + return NULL; + } - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - smlDestroyTableHandle(&smlHandle->tableExecHandle); // free for each table - SSchema* pTagsSchema = getTableTagSchema(pTableMeta); - insSetBoundColumnInfo(&smlHandle->tableExecHandle.tags, pTagsSchema, getNumOfTags(pTableMeta)); - int ret = smlBoundColumnData(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, true); + ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "bound tags error"); + return NULL; + } + return pTableCxt; +} + +int32_t smlBuildRow(STableDataCxt* pTableCxt) { + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + int ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { return ret; } - STag* pTag = NULL; - SArray* tagName = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &tagName, &pBuf); + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + return TSDB_CODE_SUCCESS; +} + +int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index) { + int ret = TSDB_CODE_SUCCESS; + SSchema* pColSchema = schema + index; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); + SSmlKv* kv = (SSmlKv*)data; + if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + int64_t size = pColSchema->bytes - VARSTR_HEADER_SIZE; + if(size <= 0){ + ret = TSDB_CODE_SML_INVALID_DATA; + goto end; + } + char* pUcs4 = taosMemoryCalloc(1, size); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, size, &len)) { + if (errno == E2BIG) { + taosMemoryFree(pUcs4); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + taosMemoryFree(pUcs4); + ret = TSDB_CODE_TSC_INVALID_VALUE; + goto end; + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; + } else { + memcpy(&pVal->value.val, &(kv->value), kv->length); + } + pVal->flag = CV_FLAG_VALUE; + +end: + return ret; +} + +int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols, + STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, + char* msgBuf, int16_t msgBufLen) { + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + + SSchema* pTagsSchema = getTableTagSchema(pTableMeta); + SBoundColInfo bindTags = {0}; + SVCreateTbReq* pCreateTblReq = NULL; + SArray* tagName = NULL; + + insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags); + int ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true); if (ret != TSDB_CODE_SUCCESS) { - taosArrayDestroy(tagName); - return ret; + buildInvalidOperationMsg(&pBuf, "bound tags error"); + goto end; } - insBuildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, - pTableMeta->tableInfo.numOfTags, ttl); - taosArrayDestroy(tagName); + STag* pTag = NULL; + + ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf); + if (ret != TSDB_CODE_SUCCESS) { + goto end; + } - smlHandle->tableExecHandle.createTblReq.ctb.stbName = taosMemoryMalloc(sTableNameLen + 1); - memcpy(smlHandle->tableExecHandle.createTblReq.ctb.stbName, sTableName, sTableNameLen); - smlHandle->tableExecHandle.createTblReq.ctb.stbName[sTableNameLen] = 0; + pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pCreateTblReq) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName, pTableMeta->tableInfo.numOfTags, + ttl); + + pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1); + memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen); + + if (dataFormat) { + STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, + &pTableMeta->uid, sizeof(pTableMeta->uid)); + if (NULL == pTableCxt) { + ret = buildInvalidOperationMsg(&pBuf, "dataformat true. get tableDataCtx error"); + goto end; + } + (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; + (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq; + (*pTableCxt)->pMeta->uid = pTableMeta->uid; + (*pTableCxt)->pMeta->vgId = pTableMeta->vgId; + pCreateTblReq = NULL; + goto end; + } - STableDataBlocks* pDataBlock = NULL; - ret = insGetDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), - TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, - pTableMeta, &pDataBlock, NULL, &smlHandle->tableExecHandle.createTblReq); + STableDataCxt* pTableCxt = NULL; + ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false); if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "create data block error"); - return ret; + buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); + goto end; } SSchema* pSchema = getTableColumnSchema(pTableMeta); - - ret = smlBoundColumnData(colsSchema, &pDataBlock->boundColumnInfo, pSchema, false); + ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "bound cols error"); - return ret; + goto end; } - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo); + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + buildInvalidOperationMsg(&pBuf, "initTableColSubmitData error"); + goto end; + } int32_t rowNum = taosArrayGetSize(cols); if (rowNum <= 0) { - return buildInvalidOperationMsg(&pBuf, "cols size <= 0"); - } - ret = insAllocateMemForSize(pDataBlock, extendedRowSize * rowNum); - if (ret != TSDB_CODE_SUCCESS) { - buildInvalidOperationMsg(&pBuf, "allocate memory error"); - return ret; + ret = buildInvalidOperationMsg(&pBuf, "cols size <= 0"); + goto end; } + for (int32_t r = 0; r < rowNum; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); - void* rowData = taosArrayGetP(cols, r); - size_t rowDataSize = 0; - if (format) { - rowDataSize = taosArrayGetSize(rowData); - } + void* rowData = taosArrayGetP(cols, r); // 1. set the parsed value from sql string - for (int c = 0, j = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; - - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - SSmlKv* kv = NULL; - if (format) { - if (j < rowDataSize) { - kv = taosArrayGetP(rowData, j); - if (rowDataSize != spd->numOfBound && j != 0 && - (kv->keyLen != strlen(pColSchema->name) || strncmp(kv->key, pColSchema->name, kv->keyLen) != 0)) { - kv = NULL; - } else { - j++; - } - } - } else { - void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); - if (p) kv = *p; + for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) { + SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]); + void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name)); + if (p == NULL) { + continue; } - - if (kv) { - int32_t colLen = kv->length; - if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { - uDebug("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); - kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); - uDebug("SML:data after:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); + SSmlKv* kv = *(SSmlKv**)p; + if(kv->type != pColSchema->type){ + ret = buildInvalidOperationMsg(&pBuf, "kv type not equal to col type"); + goto end; + } + if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { + kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision); + } + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pColSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + ret = TSDB_CODE_OUT_OF_MEMORY; + goto end; } - - if (IS_VAR_DATA_TYPE(kv->type)) { - insMemRowAppend(&pBuf, kv->value, colLen, ¶m); - } else { - insMemRowAppend(&pBuf, &(kv->value), colLen, ¶m); + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + if (errno == E2BIG) { + buildInvalidOperationMsg(&pBuf, "value too long"); + ret = TSDB_CODE_PAR_VALUE_TOO_LONG; + goto end; + } + ret = buildInvalidOperationMsg(&pBuf, strerror(errno)); + goto end; } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + pVal->value.nData = kv->length; + pVal->value.pData = (uint8_t*)kv->value; } else { - pBuilder->hasNone = true; - } - - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); + memcpy(&pVal->value.val, &(kv->value), kv->length); } + pVal->flag = CV_FLAG_VALUE; } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS != ret) { + buildInvalidOperationMsg(&pBuf, "tRowBuild error"); + goto end; } - - tdSRowEnd(pBuilder); - pDataBlock->size += extendedRowSize; + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + clearColValArray(pTableCxt->pValues); } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, rowNum, &pBuf); +end: + insDestroyBoundColInfo(&bindTags); + tdDestroySVCreateTbReq(pCreateTblReq); + taosMemoryFree(pCreateTblReq); + taosArrayDestroy(tagName); + return ret; } -void* smlInitHandle(SQuery* pQuery) { - SSmlExecHandle* handle = taosMemoryCalloc(1, sizeof(SSmlExecHandle)); - if (!handle) return NULL; - handle->pBlockHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); - handle->pQuery = pQuery; - - return handle; -} +SQuery* smlInitHandle() { + SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); + if (NULL == pQuery) { + uError("create pQuery error"); + return NULL; + } + pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; + pQuery->haveResultSet = false; + pQuery->msgType = TDMT_VND_SUBMIT; + SVnodeModifyOpStmt* stmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); + if (NULL == stmt) { + uError("create SVnodeModifyOpStmt error"); + qDestroyQuery(pQuery); + return NULL; + } + stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + stmt->freeHashFunc = insDestroyTableDataCxtHashMap; + stmt->freeArrayFunc = insDestroyVgroupDataCxtList; -void smlDestroyHandle(void* pHandle) { - if (!pHandle) return; - SSmlExecHandle* handle = (SSmlExecHandle*)pHandle; - insDestroyBlockHashmap(handle->pBlockHash); - smlDestroyTableHandle(&handle->tableExecHandle); - taosMemoryFree(handle); + pQuery->pRoot = (SNode*)stmt; + return pQuery; } -int32_t smlBuildOutput(void* handle, SHashObj* pVgHash) { - SSmlExecHandle* smlHandle = (SSmlExecHandle*)handle; - return qBuildStmtOutput(smlHandle->pQuery, pVgHash, smlHandle->pBlockHash); +int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) { + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + // merge according to vgId + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insMergeTableDataCxt failed"); + return code; + } + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + if (code != TSDB_CODE_SUCCESS) { + uError("insBuildVgDataBlocks failed"); + return code; + } + return code; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index abee9416faf733d437a34f089f74ec1b1f20e7ed..3fbe23592a8ed2344b4df8898cec9f80c5dc906d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -45,13 +45,13 @@ } while (TK_NK_SPACE == (token).type) typedef struct SInsertParseContext { - SParseContext* pComCxt; - SMsgBuf msg; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; - SParsedDataColInfo tags; // for stmt - bool missCache; - bool usingDuplicateTable; - bool forceUpdate; + SParseContext* pComCxt; + SMsgBuf msg; + char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW]; + SBoundColInfo tags; // for stmt + bool missCache; + bool usingDuplicateTable; + bool forceUpdate; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -162,7 +162,7 @@ static int32_t ignoreUsingClause(SInsertParseContext* pCxt, const char** pSql) { return code; } -static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pDuplicate) { +static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pDuplicate) { *pDuplicate = false; char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -180,21 +180,19 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifO } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, - SParsedDataColInfo* pColList, SSchema* pSchema) { - col_id_t nCols = pColList->numOfCols; - - pColList->numOfBound = 0; - pColList->boundNullLen = 0; - memset(pColList->boundColumns, 0, sizeof(col_id_t) * nCols); - for (col_id_t i = 0; i < nCols; ++i) { - pColList->cols[i].valStat = VAL_STAT_NONE; +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, SSchema* pSchema, + SBoundColInfo* pBoundInfo) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; } - SToken token; - bool isOrdered = true; - col_id_t lastColIdx = -1; // last column found - while (1) { + pBoundInfo->numOfBound = 0; + + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code) { + SToken token; NEXT_TOKEN(*pSql, token); if (TK_NK_RP == token.type) { @@ -206,64 +204,30 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, b token.z = tmpTokenBuf; token.n = strdequote(token.z); - col_id_t t = lastColIdx + 1; - col_id_t index = insFindCol(&token, t, nCols, pSchema); + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); if (index < 0 && t > 0) { index = insFindCol(&token, 0, t, pSchema); - isOrdered = false; } if (index < 0) { - return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); - } - if (pColList->cols[index].valStat == VAL_STAT_HAS) { - return buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); - } - lastColIdx = index; - pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundColumns[pColList->numOfBound] = index; - ++pColList->numOfBound; - switch (pSchema[t].type) { - case TSDB_DATA_TYPE_BINARY: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + CHAR_BYTES); - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->boundNullLen += (sizeof(VarDataOffsetT) + VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - break; - default: - pColList->boundNullLen += TYPE_BYTES[pSchema[t].type]; - break; + code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMN, token.z); + } else if (pUseCols[index]) { + code = buildSyntaxErrMsg(&pCxt->msg, "duplicated column name", token.z); + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { - return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + if (TSDB_CODE_SUCCESS == code && !isTags && !pUseCols[0]) { + code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; - - if (!isOrdered) { - pColList->colIdxInfo = taosMemoryCalloc(pColList->numOfBound, sizeof(SBoundIdxInfo)); - if (NULL == pColList->colIdxInfo) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SBoundIdxInfo* pColIdx = pColList->colIdxInfo; - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].schemaColIdx = pColList->boundColumns[i]; - pColIdx[i].boundIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insSchemaIdxCompar); - for (col_id_t i = 0; i < pColList->numOfBound; ++i) { - pColIdx[i].finalIdx = i; - } - taosSort(pColIdx, pColList->numOfBound, sizeof(SBoundIdxInfo), insBoundIdxCompar); - } - - if (pColList->numOfCols > pColList->numOfBound) { - memset(&pColList->boundColumns[pColList->numOfBound], 0, - sizeof(col_id_t) * (pColList->numOfCols - pColList->numOfBound)); - } + taosMemoryFree(pUseCols); - return TSDB_CODE_SUCCESS; + return code; } static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t* time, SMsgBuf* pMsgBuf) { @@ -485,7 +449,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - val->pData = strdup(pToken->z); + val->pData = taosStrdup(pToken->z); val->nData = pToken->n; break; } @@ -525,9 +489,8 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, // input pStmt->pSql: [(tag1_name, ...)] TAGS (tag1_value, ...) ... // output pStmt->pSql: TAGS (tag1_value, ...) ... -static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - SSchema* pTagsSchema = getTableTagSchema(pStmt->pTableMeta); - insSetBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pStmt->pTableMeta)); +static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insInitBoundColsInfo(getNumOfTags(pStmt->pTableMeta), &pCxt->tags); SToken token; int32_t index = 0; @@ -537,10 +500,10 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, getTableTagSchema(pStmt->pTableMeta), &pCxt->tags); } -static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, +static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, SArray* pTagName, SArray* pTagVals, STag** pTag) { if (!isNullValue(pTagSchema->type, pToken)) { taosArrayPush(pTagName, pTagSchema->name); @@ -568,10 +531,15 @@ static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt return code; } -static void buildCreateTbReq(SVnodeModifOpStmt* pStmt, STag* pTag, SArray* pTagName) { - insBuildCreateTbReq(&pStmt->createTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, +static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* pTagName) { + pStmt->pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pStmt->pCreateTblReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } + insBuildCreateTbReq(pStmt->pCreateTblReq, pStmt->targetTableName.tname, pTag, pStmt->pTableMeta->suid, pStmt->usingTableName.tname, pTagName, pStmt->pTableMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + return TSDB_CODE_SUCCESS; } static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { @@ -598,7 +566,7 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs } // pSql -> tag1_value, ...) -static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = getTableTagSchema(pStmt->pTableMeta); SArray* pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); @@ -625,7 +593,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* break; } - SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; + SSchema* pTagSchema = &pSchema[pCxt->tags.pColIndex[i]]; isJson = pTagSchema->type == TSDB_DATA_TYPE_JSON; code = checkAndTrimValue(&token, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { @@ -638,7 +606,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* } if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - buildCreateTbReq(pStmt, pTag, pTagName); + code = buildCreateTbReq(pStmt, pTag, pTagName); pTag = NULL; } @@ -656,7 +624,7 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* // input pStmt->pSql: TAGS (tag1_value, ...) [table_options] ... // output pStmt->pSql: [table_options] ... -static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); if (TK_TAGS != token.type) { @@ -680,7 +648,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt return code; } -static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { pStmt->pTableMeta->suid = pStmt->pTableMeta->uid; pStmt->pTableMeta->uid = pStmt->totalTbNum; pStmt->pTableMeta->tableType = TSDB_CHILD_TABLE; @@ -695,7 +663,7 @@ static int32_t storeTableMeta(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStm return taosHashPut(pStmt->pSubTableHashObj, tbFName, strlen(tbFName), &pBackup, POINTER_BYTES); } -static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { do { int32_t index = 0; SToken token; @@ -706,8 +674,8 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p if (TK_NK_INTEGER != token.type) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } - pStmt->createTblReq.ttl = taosStr2Int32(token.z, NULL, 10); - if (pStmt->createTblReq.ttl < 0) { + pStmt->pCreateTblReq->ttl = taosStr2Int32(token.z, NULL, 10); + if (pStmt->pCreateTblReq->ttl < 0) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", token.z); } } else if (TK_COMMENT == token.type) { @@ -720,11 +688,11 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return buildSyntaxErrMsg(&pCxt->msg, "comment too long", token.z); } int32_t len = trimString(token.z, token.n, pCxt->tmpTokenBuf, TSDB_TB_COMMENT_LEN); - pStmt->createTblReq.comment = strndup(pCxt->tmpTokenBuf, len); - if (NULL == pStmt->createTblReq.comment) { + pStmt->pCreateTblReq->comment = strndup(pCxt->tmpTokenBuf, len); + if (NULL == pStmt->pCreateTblReq->comment) { return TSDB_CODE_OUT_OF_MEMORY; } - pStmt->createTblReq.commentLen = len; + pStmt->pCreateTblReq->commentLen = len; } else { break; } @@ -738,7 +706,7 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt, SVnodeModifOpStmt* p // output pStmt->pSql: // 1. [(field1_name, ...)] // 2. VALUES ... | FILE ... -static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (!pStmt->usingTableProcessing || pCxt->usingDuplicateTable) { return TSDB_CODE_SUCCESS; } @@ -812,7 +780,7 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt return code; } -static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool isStb, bool* pMissCache) { +static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool isStb, bool* pMissCache) { int32_t code = TSDB_CODE_SUCCESS; SVgroupInfo vg; bool exists = true; @@ -837,7 +805,7 @@ static int32_t getTableVgroup(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, boo return code; } -static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { +static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SVgroupInfo vg; int32_t code = catalogGetCachedTableVgMeta(pCxt->pCatalog, &pStmt->targetTableName, &vg, &pStmt->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -849,7 +817,7 @@ static int32_t getTableMetaAndVgroupImpl(SParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, bool* pMissCache) { +static int32_t getTableMetaAndVgroup(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { @@ -875,7 +843,7 @@ static int32_t collectUseDatabase(const SName* pName, SHashObj* pDbs) { return taosHashPut(pDbs, dbFName, strlen(dbFName), dbFName, sizeof(dbFName)); } -static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pCxt->forceUpdate) { pCxt->missCache = true; return TSDB_CODE_SUCCESS; @@ -894,11 +862,11 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt return code; } -static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t preParseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { return insCreateSName(&pStmt->usingTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); } -static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pCxt->forceUpdate) { pCxt->missCache = true; return TSDB_CODE_SUCCESS; @@ -920,7 +888,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); int32_t code = preParseUsingTableName(pCxt, pStmt, &token); @@ -939,7 +907,7 @@ static int32_t parseUsingTableNameImpl(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [(tag1_name, ...)] TAGS (tag1_value, ...) [table_options]] ... // 2. VALUES ... | FILE ... -static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -957,7 +925,7 @@ static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* return code; } -static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); } @@ -968,7 +936,7 @@ static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [ USING ... ] ... // 2. VALUES ... | FILE ... -static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -982,26 +950,22 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif return skipParentheses(pCxt, &pStmt->pSql); } -static int32_t getTableDataBlocks(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks** pDataBuf) { +static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { - uint64_t uid = pStmt->pTableMeta->uid; - if (pStmt->usingTableProcessing) { - pStmt->pTableMeta->uid = 0; - } - - return insGetDataBlockFromList( - pStmt->pTableBlockHashObj, &uid, sizeof(pStmt->pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), - getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, pDataBuf, NULL, &pStmt->createTblReq); + return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false); } + char tbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStmt->targetTableName, tbFName); - return insGetDataBlockFromList(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), getTableInfo(pStmt->pTableMeta).rowSize, pStmt->pTableMeta, - pDataBuf, NULL, &pStmt->createTblReq); + if (pStmt->usingTableProcessing) { + pStmt->pTableMeta->uid = 0; + } + return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb); } -static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, - STableDataBlocks* pDataBuf) { +static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; int32_t index = 0; NEXT_TOKEN_KEEP_SQL(pStmt->pSql, token, index); @@ -1011,13 +975,30 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, - getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, getTableColumnSchema(pStmt->pTableMeta), + &pTableCxt->boundColsInfo); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t initTableColSubmitData(STableDataCxt* pTableCxt) { + if (0 == (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < pTableCxt->boundColsInfo.numOfBound; ++i) { + SSchema* pSchema = &pTableCxt->pMeta->schema[pTableCxt->boundColsInfo.pColIndex[i]]; + SColData* pCol = taosArrayReserve(pTableCxt->pData->aCol, 1); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + tColDataInit(pCol, pSchema->colId, pSchema->type, 0); } return TSDB_CODE_SUCCESS; @@ -1027,14 +1008,17 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... // output pStmt->pSql: VALUES ... | FILE ... -static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, - STableDataBlocks** pDataBuf) { +static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, + STableDataCxt** pTableCxt) { int32_t code = parseUsingClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { - code = getTableDataBlocks(pCxt, pStmt, pDataBuf); + code = getTableDataCxt(pCxt, pStmt, pTableCxt); + } + if (TSDB_CODE_SUCCESS == code) { + code = parseBoundColumnsClause(pCxt, pStmt, *pTableCxt); } if (TSDB_CODE_SUCCESS == code) { - code = parseBoundColumnsClause(pCxt, pStmt, *pDataBuf); + code = initTableColSubmitData(*pTableCxt); } return code; } @@ -1043,7 +1027,7 @@ static int32_t parseSchemaClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpS // output pStmt->pSql: // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... -static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { int32_t code = preParseTargetTableName(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code) { // option: [(field1_name, ...)] @@ -1057,108 +1041,88 @@ static int32_t parseSchemaClauseTop(SInsertParseContext* pCxt, SVnodeModifOpStmt } static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { - int64_t iv; - uint64_t uv; - char* endptr = NULL; - + int16_t timePrec, SColVal* pVal) { switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { if (strncmp(pToken->z, "true", pToken->n) == 0) { - return func(&pCxt->msg, &TRUE_VALUE, pSchema->bytes, param); + pVal->value.val = TRUE_VALUE; } else if (strncmp(pToken->z, "false", pToken->n) == 0) { - return func(&pCxt->msg, &FALSE_VALUE, pSchema->bytes, param); + pVal->value.val = FALSE_VALUE; } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } } else if (pToken->type == TK_NK_INTEGER) { - return func(&pCxt->msg, ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? &FALSE_VALUE : &TRUE_VALUE), - pSchema->bytes, param); + pVal->value.val = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); } else if (pToken->type == TK_NK_FLOAT) { - return func(&pCxt->msg, ((taosStr2Double(pToken->z, NULL) == 0) ? &FALSE_VALUE : &TRUE_VALUE), pSchema->bytes, - param); + pVal->value.val = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); } else { return buildSyntaxErrMsg(&pCxt->msg, "invalid bool data", pToken->z); } + break; } - case TSDB_DATA_TYPE_TINYINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid tinyint data", pToken->z); - } else if (!IS_VALID_TINYINT(iv)) { + } else if (!IS_VALID_TINYINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "tinyint data overflow", pToken->z); } - - uint8_t tmpVal = (uint8_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UTINYINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned tinyint data", pToken->z); - } else if (uv > UINT8_MAX) { + } else if (pVal->value.val > UINT8_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned tinyint data overflow", pToken->z); } - uint8_t tmpVal = (uint8_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_SMALLINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid smallint data", pToken->z); - } else if (!IS_VALID_SMALLINT(iv)) { + } else if (!IS_VALID_SMALLINT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "smallint data overflow", pToken->z); } - int16_t tmpVal = (int16_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_USMALLINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned smallint data", pToken->z); - } else if (uv > UINT16_MAX) { + } else if (pVal->value.val > UINT16_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned smallint data overflow", pToken->z); } - uint16_t tmpVal = (uint16_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_INT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid int data", pToken->z); - } else if (!IS_VALID_INT(iv)) { + } else if (!IS_VALID_INT(pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "int data overflow", pToken->z); } - int32_t tmpVal = (int32_t)iv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned int data", pToken->z); - } else if (uv > UINT32_MAX) { + } else if (pVal->value.val > UINT32_MAX) { return buildSyntaxErrMsg(&pCxt->msg, "unsigned int data overflow", pToken->z); } - uint32_t tmpVal = (uint32_t)uv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_BIGINT: { - if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid bigint data", pToken->z); } - return func(&pCxt->msg, &iv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_UBIGINT: { - if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &pVal->value.val)) { return buildSyntaxErrMsg(&pCxt->msg, "invalid unsigned bigint data", pToken->z); } - return func(&pCxt->msg, &uv, pSchema->bytes, param); + break; } - case TSDB_DATA_TYPE_FLOAT: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); @@ -1167,11 +1131,12 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal float data", pToken->z); } - float tmpVal = (float)dv; - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + float f = dv; + memcpy(&pVal->value.val, &f, sizeof(f)); + break; } - case TSDB_DATA_TYPE_DOUBLE: { + char* endptr = NULL; double dv; if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); @@ -1179,49 +1144,77 @@ static int32_t parseValueTokenImpl(SInsertParseContext* pCxt, const char** pSql, if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { return buildSyntaxErrMsg(&pCxt->msg, "illegal double data", pToken->z); } - return func(&pCxt->msg, &dv, pSchema->bytes, param); + pVal->value.val = *(int64_t*)&dv; + break; } - case TSDB_DATA_TYPE_BINARY: { // Too long values will raise the invalid sql error message if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } - - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } - case TSDB_DATA_TYPE_NCHAR: { - return func(&pCxt->msg, pToken->z, pToken->n, param); + // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + int32_t len = 0; + char* pUcs4 = taosMemoryCalloc(1, pSchema->bytes - VARSTR_HEADER_SIZE); + if (NULL == pUcs4) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)pUcs4, pSchema->bytes - VARSTR_HEADER_SIZE, &len)) { + taosMemoryFree(pUcs4); + if (errno == E2BIG) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(&pCxt->msg, buf, pToken->z); + } + pVal->value.pData = pUcs4; + pVal->value.nData = len; + break; } case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", pToken->z); } - return func(&pCxt->msg, pToken->z, pToken->n, param); + pVal->value.pData = taosMemoryMalloc(pToken->n); + if (NULL == pVal->value.pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pVal->value.pData, pToken->z, pToken->n); + pVal->value.nData = pToken->n; + break; } case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t tmpVal; - if (parseTime(pSql, pToken, timePrec, &tmpVal, &pCxt->msg) != TSDB_CODE_SUCCESS) { + if (parseTime(pSql, pToken, timePrec, &pVal->value.val, &pCxt->msg) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid timestamp", pToken->z); } - - return func(&pCxt->msg, &tmpVal, pSchema->bytes, param); + break; } + default: + return TSDB_CODE_FAILED; } - return TSDB_CODE_FAILED; + pVal->flag = CV_FLAG_VALUE; + return TSDB_CODE_SUCCESS; } static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, SToken* pToken, SSchema* pSchema, - int16_t timePrec, _row_append_fn_t func, void* param) { + int16_t timePrec, SColVal* pVal) { int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (TSDB_CODE_SUCCESS == code && isNullValue(pSchema->type, pToken)) { if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return buildSyntaxErrMsg(&pCxt->msg, "primary timestamp should not be null", pToken->z); } - - return func(&pCxt->msg, NULL, 0, param); + pVal->flag = CV_FLAG_NULL; + return TSDB_CODE_SUCCESS; } if (TSDB_CODE_SUCCESS == code && IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { @@ -1229,22 +1222,29 @@ static int32_t parseValueToken(SInsertParseContext* pCxt, const char** pSql, STo } if (TSDB_CODE_SUCCESS == code) { - code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, func, param); + code = parseValueTokenImpl(pCxt, pSql, pToken, pSchema, timePrec, pVal); } return code; } -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataBlocks* pDataBuf, bool* pGotRow, +static void clearColValArray(SArray* pCols) { + int32_t num = taosArrayGetSize(pCols); + for (int32_t i = 0; i < num; ++i) { + SColVal* pCol = taosArrayGet(pCols, i); + if (IS_VAR_DATA_TYPE(pCol->type)) { + taosMemoryFreeClear(pCol->value.pData); + } + } +} + +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { - SRowBuilder* pBuilder = &pDataBuf->rowBuilder; - STSRow* row = (STSRow*)(pDataBuf->pData + pDataBuf->size); // skip the SSubmitBlk header - SParsedDataColInfo* pCols = &pDataBuf->boundColumnInfo; - bool isParseBindParam = false; - SSchema* pSchemas = getTableColumnSchema(pDataBuf->pTableMeta); - SMemParam param = {.rb = pBuilder}; - - int32_t code = tdSRowResetBuf(pBuilder, row); + SBoundColInfo* pCols = &pTableCxt->boundColsInfo; + bool isParseBindParam = false; + SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); + + int32_t code = TSDB_CODE_SUCCESS; // 1. set the parsed value from sql string for (int i = 0; i < pCols->numOfBound && TSDB_CODE_SUCCESS == code; ++i) { const char* pOrigSql = *pSql; @@ -1255,6 +1255,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB break; } + SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; + SColVal* pVal = taosArrayGet(pTableCxt->pValues, pCols->pColIndex[i]); + if (pToken->type == TK_NK_QUESTION) { isParseBindParam = true; if (NULL == pCxt->pComCxt->pStmtCb) { @@ -1272,10 +1275,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB break; } - param.schema = &pSchemas[pCols->boundColumns[i]]; - insGetSTSRowAppendInfo(pBuilder->rowType, pCols, i, ¶m.toffset, ¶m.colIdx); - code = parseValueToken(pCxt, pSql, pToken, param.schema, getTableInfo(pDataBuf->pTableMeta).precision, - insMemRowAppend, ¶m); + if (TSDB_CODE_SUCCESS == code) { + code = parseValueToken(pCxt, pSql, pToken, pSchema, getTableInfo(pTableCxt->pMeta).precision, pVal); + } } if (TSDB_CODE_SUCCESS == code && i < pCols->numOfBound - 1) { @@ -1286,65 +1288,28 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataB } } - if (TSDB_CODE_SUCCESS == code) { - TSKEY tsKey = TD_ROW_KEY(row); - code = insCheckTimestamp(pDataBuf, (const char*)&tsKey); - } - if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { - // set the null value for the columns that do not assign values - if ((pCols->numOfBound < pCols->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); + code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); + if (TSDB_CODE_SUCCESS == code) { + insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); } + } - tdSRowEnd(pBuilder); - + if (TSDB_CODE_SUCCESS == code && !isParseBindParam) { *pGotRow = true; - -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif } - return code; -} + clearColValArray(pTableCxt->pValues); -static int32_t allocateMemIfNeed(STableDataBlocks* pDataBlock, int32_t rowSize, int32_t* numOfRows) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - const int factor = 5; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < rowSize * factor) { - while (remain < rowSize * factor) { - pDataBlock->nAllocSize = (uint32_t)(pDataBlock->nAllocSize * 1.5); - remain = pDataBlock->nAllocSize - pDataBlock->size; - } - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - *numOfRows = (int32_t)(pDataBlock->nAllocSize - pDataBlock->headerSize) / rowSize; - return TSDB_CODE_SUCCESS; + return code; } // pSql -> (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, - int32_t maxRows, int32_t* pNumOfRows, SToken* pToken) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); +static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows, SToken* pToken) { + int32_t code = TSDB_CODE_SUCCESS; - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); (*pNumOfRows) = 0; while (TSDB_CODE_SUCCESS == code) { int32_t index = 0; @@ -1354,13 +1319,9 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } pStmt->pSql += index; - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { - code = parseOneRow(pCxt, &pStmt->pSql, pDataBuf, &gotRow, pToken); + code = parseOneRow(pCxt, &pStmt->pSql, pTableCxt, &gotRow, pToken); } if (TSDB_CODE_SUCCESS == code) { @@ -1373,7 +1334,6 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } } @@ -1386,19 +1346,11 @@ static int32_t parseValues(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } // VALUES (field1_value, ...) [(field1_value2, ...) ...] -static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { - int32_t maxNumOfRows = 0; int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseValues(pCxt, pStmt, pTableCxt, &numOfRows, pToken); if (TSDB_CODE_SUCCESS == code) { - code = parseValues(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows, pToken); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_INSERT); @@ -1406,11 +1358,9 @@ static int32_t parseValuesClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return code; } -static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, - int maxRows, int32_t* pNumOfRows) { - int32_t code = insInitRowBuilder(&pDataBuf->rowBuilder, pDataBuf->pTableMeta->sversion, &pDataBuf->boundColumnInfo); - - int32_t extendedRowSize = insGetExtendedRowSize(pDataBuf); +static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, + int32_t* pNumOfRows) { + int32_t code = TSDB_CODE_SUCCESS; (*pNumOfRows) = 0; char* pLine = NULL; int64_t readLen = 0; @@ -1426,16 +1376,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, continue; } - if ((*pNumOfRows) >= maxRows || pDataBuf->size + extendedRowSize >= pDataBuf->nAllocSize) { - code = allocateMemIfNeed(pDataBuf, extendedRowSize, &maxRows); - } - bool gotRow = false; if (TSDB_CODE_SUCCESS == code) { SToken token; strtolower(pLine, pLine); const char* pRow = pLine; - code = parseOneRow(pCxt, (const char**)&pRow, pDataBuf, &gotRow, &token); + + code = parseOneRow(pCxt, (const char**)&pRow, pTableCxt, &gotRow, &token); if (code && firstLine) { firstLine = false; code = 0; @@ -1444,11 +1391,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, } if (TSDB_CODE_SUCCESS == code && gotRow) { - pDataBuf->size += extendedRowSize; (*pNumOfRows)++; } - if (TSDB_CODE_SUCCESS == code && pDataBuf->nAllocSize > tsMaxMemUsedByInsert * 1024 * 1024) { + if (TSDB_CODE_SUCCESS == code && (*pNumOfRows) > tsMaxMemUsedByInsert * 1024 * 1024) { pStmt->fileProcessing = true; break; } @@ -1464,18 +1410,10 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { - int32_t maxNumOfRows = 0; +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { int32_t numOfRows = 0; - int32_t code = allocateMemIfNeed(pDataBuf, insGetExtendedRowSize(pDataBuf), &maxNumOfRows); + int32_t code = parseCsvFile(pCxt, pStmt, pTableCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { - code = parseCsvFile(pCxt, pStmt, pDataBuf, maxNumOfRows, &numOfRows); - } - if (TSDB_CODE_SUCCESS == code) { - code = insSetBlockInfo((SSubmitBlk*)(pDataBuf->pData), pDataBuf, numOfRows, &pCxt->msg); - } - if (TSDB_CODE_SUCCESS == code) { - pDataBuf->numOfTables = 1; pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); @@ -1488,8 +1426,8 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifOpStm return code; } -static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pFilePath, - STableDataBlocks* pDataBuf) { +static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pFilePath, + STableDataCxt* pTableCxt) { char filePathStr[TSDB_FILENAME_LEN] = {0}; if (TK_NK_STRING == pFilePath->type) { trimString(pFilePath->z, pFilePath->n, filePathStr, sizeof(filePathStr)); @@ -1501,10 +1439,10 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SVnodeModifOpStmt* p return TAOS_SYSTEM_ERROR(errno); } - return parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + return parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } -static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf, +static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt, SToken* pToken) { if (tsUseAdapter) { return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); @@ -1514,18 +1452,18 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); } - return parseDataFromFile(pCxt, pStmt, pToken, pDataBuf); + return parseDataFromFile(pCxt, pStmt, pToken, pTableCxt); } // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path -static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, STableDataBlocks* pDataBuf) { +static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { SToken token; NEXT_TOKEN(pStmt->pSql, token); switch (token.type) { case TK_VALUES: - return parseValuesClause(pCxt, pStmt, pDataBuf, &token); + return parseValuesClause(pCxt, pStmt, pTableCxt, &token); case TK_FILE: - return parseFileClause(pCxt, pStmt, pDataBuf, &token); + return parseFileClause(pCxt, pStmt, pTableCxt, &token); default: break; } @@ -1535,19 +1473,20 @@ static int32_t parseDataClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt // input pStmt->pSql: // 1. [(tag1_name, ...)] ... // 2. VALUES ... | FILE ... -static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pDataBuf); +static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataClause(pCxt, pStmt, pDataBuf); + code = parseDataClause(pCxt, pStmt, pTableCxt); } return code; } -static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - destroyBoundColumnInfo(&pCxt->tags); +static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); - tdDestroySVCreateTbReq(&pStmt->createTblReq); + tdDestroySVCreateTbReq(pStmt->pCreateTblReq); + taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; pCxt->usingDuplicateTable = false; pStmt->pBoundCols = NULL; @@ -1556,7 +1495,7 @@ static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt } // input pStmt->pSql: [(field1_name, ...)] [ USING ... ] VALUES ... | FILE ... -static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName) { +static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { resetEnvPreTable(pCxt, pStmt); int32_t code = parseSchemaClauseTop(pCxt, pStmt, pTbName); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { @@ -1565,7 +1504,7 @@ static int32_t parseInsertTableClause(SInsertParseContext* pCxt, SVnodeModifOpSt return code; } -static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SToken* pTbName, +static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName, bool* pHasData) { // no data in the sql string anymore. if (0 == pTbName->n) { @@ -1604,8 +1543,8 @@ static int32_t checkTableClauseFirstToken(SInsertParseContext* pCxt, SVnodeModif return TSDB_CODE_SUCCESS; } -static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - SParsedDataColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); +static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + SBoundColInfo* tags = taosMemoryMalloc(sizeof(pCxt->tags)); if (NULL == tags) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1622,19 +1561,17 @@ static int32_t setStmtInfo(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) return code; } -static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (TSDB_QUERY_HAS_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { return setStmtInfo(pCxt, pStmt); } // merge according to vgId - int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStmt->pTableBlockHashObj) > 0) { - code = insMergeTableDataBlocks(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); - } + int32_t code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks); if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); + code = insBuildVgDataBlocks(pStmt->pVgroupsHashObj, pStmt->pVgDataBlocks, &pStmt->pDataBlocks); } + return code; } @@ -1643,7 +1580,7 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifOpStm // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; -static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { SToken token; int32_t code = TSDB_CODE_SUCCESS; bool hasData = true; @@ -1666,7 +1603,7 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); if (NULL == pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1675,8 +1612,8 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } pStmt->pSql = pCxt->pComCxt->pSql; - pStmt->freeHashFunc = insDestroyBlockHashmap; - pStmt->freeArrayFunc = insDestroyBlockArrayList; + pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; + pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); @@ -1745,7 +1682,7 @@ static int32_t getTableMetaFromMetaData(const SArray* pTables, STableMeta** pMet return pRes->code; } -static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpStmt* pStmt, bool isStb) { +static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpStmt* pStmt, bool isStb) { if (1 != taosArrayGetSize(pTables)) { return TSDB_CODE_FAILED; } @@ -1764,7 +1701,7 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt } static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, - SVnodeModifOpStmt* pStmt, bool isStb) { + SVnodeModifyOpStmt* pStmt, bool isStb) { int32_t code = checkAuthFromMetaData(pMetaData->pUser); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); @@ -1800,7 +1737,7 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { } static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, - SVnodeModifOpStmt* pStmt) { + SVnodeModifyOpStmt* pStmt) { clearCatalogReq(pCatalogReq); if (pStmt->usingTableProcessing) { @@ -1814,7 +1751,7 @@ static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); @@ -1843,7 +1780,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR return resetVnodeModifOpStmt(pCxt, *pQuery); } - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)(*pQuery)->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(*pQuery)->pRoot; if (!pStmt->fileProcessing) { return setVnodeModifOpStmt(pCxt, pCatalogReq, pMetaData, pStmt); @@ -1853,7 +1790,7 @@ static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogR } static int32_t setRefreshMate(SQuery* pQuery) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; if (taosHashGetSize(pStmt->pTableNameHashObj) > 0) { taosArrayDestroy(pQuery->pTableList); @@ -1884,7 +1821,7 @@ static int32_t setRefreshMate(SQuery* pQuery) { // [(field1_name, ...)] // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; -static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = skipInsertInto(&pStmt->pSql, &pCxt->msg); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(pCxt, pStmt); @@ -1892,11 +1829,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifOpS return code; } -static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { - STableDataBlocks* pDataBuf = NULL; - int32_t code = getTableDataBlocks(pCxt, pStmt, &pDataBuf); +static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { + STableDataCxt* pTableCxt = NULL; + int32_t code = getTableDataCxt(pCxt, pStmt, &pTableCxt); if (TSDB_CODE_SUCCESS == code) { - code = parseDataFromFileImpl(pCxt, pStmt, pDataBuf); + code = parseDataFromFileImpl(pCxt, pStmt, pTableCxt); } if (TSDB_CODE_SUCCESS == code) { @@ -1910,7 +1847,7 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifOpStm return code; } -static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = parseInsertTableClauseBottom(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(pCxt, pStmt); @@ -1918,7 +1855,7 @@ static int32_t parseInsertSqlFromTable(SInsertParseContext* pCxt, SVnodeModifOpS return code; } -static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt) { +static int32_t parseInsertSqlImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { if (pStmt->pSql == pCxt->pComCxt->pSql || NULL != pCxt->pComCxt->pStmtCb) { return parseInsertSqlFromStart(pCxt, pStmt); } @@ -1970,7 +1907,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** return TSDB_CODE_SUCCESS; } -static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SCatalogReq* pCatalogReq) { +static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) { int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); if (TSDB_CODE_SUCCESS == code) { if (0 == pStmt->usingTableName.type) { @@ -1986,7 +1923,7 @@ static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifOpStm } static int32_t setNextStageInfo(SInsertParseContext* pCxt, SQuery* pQuery, SCatalogReq* pCatalogReq) { - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; if (pCxt->missCache) { parserDebug("0x%" PRIx64 " %d rows of %d tables have been inserted before cache miss", pCxt->pComCxt->requestId, pStmt->totalRowsNum, pStmt->totalTbNum); @@ -2011,7 +1948,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { - code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); + code = parseInsertSqlImpl(&context, (SVnodeModifyOpStmt*)(*pQuery)->pRoot); } if (TSDB_CODE_SUCCESS == code) { code = setNextStageInfo(&context, *pQuery, pCatalogReq); @@ -2020,6 +1957,6 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal QUERY_EXEC_STAGE_SCHEDULE == (*pQuery)->execStage) { code = setRefreshMate(*pQuery); } - destroyBoundColumnInfo(&context.tags); + insDestroyBoundColInfo(&context.tags); return code; } diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index cfdbbbbc3663263af7af8231e3ae863af657d995..01a635e4b2d34bbb7982d45400429ac323e9547e 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -29,26 +29,53 @@ typedef struct SKvParam { char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; +int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { + *pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == *pData) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SSubmitTbData* pNew = *pData; + + *pNew = *pDataBlock->pData; + + cloneSVreateTbReq(pDataBlock->pData->pCreateTbReq, &pNew->pCreateTbReq); + pNew->aCol = taosArrayDup(pDataBlock->pData->aCol, NULL); + + int32_t colNum = taosArrayGetSize(pNew->aCol); + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pNew->aCol, i); + tColDataDeepClear(pCol); + } + + return TSDB_CODE_SUCCESS; +} + int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVgDataBlocks = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SArray* pVgDataBlocks = NULL; + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + // merge according to vgId if (taosHashGetSize(pBlockHash) > 0) { - code = insMergeTableDataBlocks(pBlockHash, &pVgDataBlocks); + code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks); } if (TSDB_CODE_SUCCESS == code) { - code = insBuildOutput(pVgHash, pVgDataBlocks, &((SVnodeModifOpStmt*)pQuery->pRoot)->pDataBlocks); + code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); + } + + if (pStmt->freeArrayFunc) { + pStmt->freeArrayFunc(pVgDataBlocks); } - insDestroyBlockArrayList(pVgDataBlocks); return code; } int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const char* sTableName, char* tName, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t code = TSDB_CODE_SUCCESS; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t code = TSDB_CODE_SUCCESS; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } @@ -64,7 +91,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); bool isJson = false; STag* pTag = NULL; @@ -74,7 +101,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch continue; } - SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; + SSchema* pTagSchema = &pSchema[tags->pColIndex[c]]; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; @@ -136,11 +163,16 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch goto end; } - SVCreateTbReq tbReq = {0}; - insBuildCreateTbReq(&tbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pTableMeta->tableInfo.numOfTags, - TSDB_DEFAULT_TABLE_TTL); - code = insBuildCreateTbMsg(pDataBlock, &tbReq); - tdDestroySVCreateTbReq(&tbReq); + if (NULL == pDataBlock->pData->pCreateTbReq) { + pDataBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pDataBlock->pData->pCreateTbReq) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + } + + insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, + pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { @@ -155,199 +187,178 @@ end: return code; } -int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - int32_t rowNum = bind->num; - - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); +int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* src, TAOS_MULTI_BIND* dst) { + int32_t output = 0; + int32_t newBuflen = (pSchema->bytes - VARSTR_HEADER_SIZE) * src->num; + if (dst->buffer_length < newBuflen) { + dst->buffer = taosMemoryRealloc(dst->buffer, newBuflen); + if (NULL == dst->buffer) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); + if (NULL == dst->length) { + dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); + if (NULL == dst->buffer) { + taosMemoryFreeClear(dst->buffer); + return TSDB_CODE_OUT_OF_MEMORY; + } + } - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size); // skip the SSubmitBlk header - tdSRowResetBuf(pBuilder, row); + dst->buffer_length = pSchema->bytes - VARSTR_HEADER_SIZE; - for (int c = 0; c < spd->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[spd->boundColumns[c]]; + for (int32_t i = 0; i < src->num; ++i) { + if (src->is_null && src->is_null[i]) { + continue; + } - if (bind[c].num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + if (!taosMbsToUcs4(((char*)src->buffer) + src->buffer_length * i, src->length[i], + (TdUcs4*)(((char*)dst->buffer) + dst->buffer_length * i), dst->buffer_length, &output)) { + if (errno == E2BIG) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), "%s", strerror(errno)); + return buildSyntaxErrMsg(pMsgBuf, buf, NULL); + } - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, c, ¶m.toffset, ¶m.colIdx); - - if (bind[c].is_null && bind[c].is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } - - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind[c].buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } + dst->length[i] = output; + } - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind[c].length[r]; - } + dst->buffer_type = src->buffer_type; + dst->is_null = src->is_null; + dst->num = src->num; - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind[c].buffer + bind[c].buffer_length * r, colLen, ¶m)); - } + return TSDB_CODE_SUCCESS; +} - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } - } - // set the null value for the columns that do not assign values - if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + int32_t rowNum = bind->num; + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, c); + + if (bind[c].num != rowNum) { + code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + goto _return; } - tdSRowEnd(pBuilder); -#ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); -#endif - pDataBlock->size += extendedRowSize; - } - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - return insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf); -} + if (bind[c].buffer_type != pColSchema->type) { + code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + goto _return; + } -int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = insGetExtendedRowSize(pDataBlock); - SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; - SRowBuilder* pBuilder = &pDataBlock->rowBuilder; - SMemParam param = {.rb = pBuilder}; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; - bool rowStart = (0 == colIdx); - bool rowEnd = ((colIdx + 1) == spd->numOfBound); - - if (rowStart) { - CHECK_CODE( - insInitRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo)); - CHECK_CODE(insAllocateMemForSize(pDataBlock, extendedRowSize * bind->num)); - } - - for (int32_t r = 0; r < bind->num; ++r) { - STSRow* row = (STSRow*)(pDataBlock->pData + pDataBlock->size + extendedRowSize * r); // skip the SSubmitBlk header - if (rowStart) { - tdSRowResetBuf(pBuilder, row); + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind + c, &ncharBind); + if (code) { + goto _return; + } + pBind = &ncharBind; } else { - tdSRowGetBuf(pBuilder, row); + pBind = bind + c; } - SSchema* pColSchema = &pSchema[spd->boundColumns[colIdx]]; - - if (bind->num != rowNum) { - return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); - } + tColDataAddValueByBind(pCol, pBind); + } - param.schema = pColSchema; - insGetSTSRowAppendInfo(pBuilder->rowType, spd, colIdx, ¶m.toffset, ¶m.colIdx); + qDebug("stmt all %d columns bind %d rows data", boundInfo->numOfBound, rowNum); - if (bind->is_null && bind->is_null[r]) { - if (pColSchema->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - return buildInvalidOperationMsg(&pBuf, "primary timestamp should not be NULL"); - } +_return: - CHECK_CODE(insMemRowAppend(&pBuf, NULL, 0, ¶m)); - } else { - if (bind->buffer_type != pColSchema->type) { - return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); - } + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); - int32_t colLen = pColSchema->bytes; - if (IS_VAR_DATA_TYPE(pColSchema->type)) { - colLen = bind->length[r]; - } + return code; +} - CHECK_CODE(insMemRowAppend(&pBuf, (char*)bind->buffer + bind->buffer_length * r, colLen, ¶m)); - } +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[colIdx]]; + SColData* pCol = taosArrayGet(pDataBlock->pData->aCol, colIdx); + TAOS_MULTI_BIND ncharBind = {0}; + TAOS_MULTI_BIND* pBind = NULL; + int32_t code = 0; + + if (bind->num != rowNum) { + return buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); + } - if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { - TSKEY tsKey = TD_ROW_KEY(row); - insCheckTimestamp(pDataBlock, (const char*)&tsKey); - } + if (bind->buffer_type != pColSchema->type) { + return buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); + } - // set the null value for the columns that do not assign values - if (rowEnd && (spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) { - pBuilder->hasNone = true; + if (TSDB_DATA_TYPE_NCHAR == pColSchema->type) { + code = convertStmtNcharCol(&pBuf, pColSchema, bind, &ncharBind); + if (code) { + goto _return; } - if (rowEnd) { - tdSRowEnd(pBuilder); - } -#ifdef TD_DEBUG_PRINT_ROW - if (rowEnd) { - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); - tdSRowPrint(row, pSTSchema, __func__); - taosMemoryFree(pSTSchema); - } -#endif + pBind = &ncharBind; + } else { + pBind = bind; } - if (rowEnd) { - pDataBlock->size += extendedRowSize * bind->num; + tColDataAddValueByBind(pCol, pBind); - SSubmitBlk* pBlocks = (SSubmitBlk*)(pDataBlock->pData); - CHECK_CODE(insSetBlockInfo(pBlocks, pDataBlock, bind->num, &pBuf)); - } + qDebug("stmt col %d bind %d rows data", colIdx, rowNum); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(ncharBind.buffer); + taosMemoryFree(ncharBind.length); + + return code; } -int32_t buildBoundFields(SParsedDataColInfo* boundInfo, SSchema* pSchema, int32_t* fieldNum, TAOS_FIELD_E** fields, - uint8_t timePrec) { +int32_t buildBoundFields(int32_t numOfBound, int16_t* boundColumns, SSchema* pSchema, int32_t* fieldNum, + TAOS_FIELD_E** fields, uint8_t timePrec) { if (fields) { - *fields = taosMemoryCalloc(boundInfo->numOfBound, sizeof(TAOS_FIELD)); + *fields = taosMemoryCalloc(numOfBound, sizeof(TAOS_FIELD_E)); if (NULL == *fields) { return TSDB_CODE_OUT_OF_MEMORY; } - SSchema* schema = &pSchema[boundInfo->boundColumns[0]]; + SSchema* schema = &pSchema[boundColumns[0]]; if (TSDB_DATA_TYPE_TIMESTAMP == schema->type) { (*fields)[0].precision = timePrec; } - for (int32_t i = 0; i < boundInfo->numOfBound; ++i) { - schema = &pSchema[boundInfo->boundColumns[i]]; + for (int32_t i = 0; i < numOfBound; ++i) { + schema = &pSchema[boundColumns[i]]; strcpy((*fields)[i].name, schema->name); (*fields)[i].type = schema->type; (*fields)[i].bytes = schema->bytes; } } - *fieldNum = boundInfo->numOfBound; + *fieldNum = numOfBound; return TSDB_CODE_SUCCESS; } int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SBoundColInfo* tags = (SBoundColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_APP_ERROR; } - if (pDataBlock->pTableMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pTableMeta->tableType != TSDB_CHILD_TABLE) { + if (pDataBlock->pMeta->tableType != TSDB_SUPER_TABLE && pDataBlock->pMeta->tableType != TSDB_CHILD_TABLE) { return TSDB_CODE_TSC_STMT_API_ERROR; } - SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); + SSchema* pSchema = getTableTagSchema(pDataBlock->pMeta); if (tags->numOfBound <= 0) { *fieldNum = 0; *fields = NULL; @@ -355,15 +366,15 @@ int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TA return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(tags, pSchema, fieldNum, fields, 0)); + CHECK_CODE(buildBoundFields(tags->numOfBound, tags->pColIndex, pSchema, fieldNum, fields, 0)); return TSDB_CODE_SUCCESS; } int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fields) { - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - if (pDataBlock->boundColumnInfo.numOfBound <= 0) { + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); + if (pDataBlock->boundColsInfo.numOfBound <= 0) { *fieldNum = 0; if (fields) { *fields = NULL; @@ -372,128 +383,125 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel return TSDB_CODE_SUCCESS; } - CHECK_CODE(buildBoundFields(&pDataBlock->boundColumnInfo, pSchema, fieldNum, fields, - pDataBlock->pTableMeta->tableInfo.precision)); + CHECK_CODE(buildBoundFields(pDataBlock->boundColsInfo.numOfBound, pDataBlock->boundColsInfo.pColIndex, pSchema, + fieldNum, fields, pDataBlock->pMeta->tableInfo.precision)); return TSDB_CODE_SUCCESS; } -int32_t qResetStmtDataBlock(void* block, bool keepBuf) { - STableDataBlocks* pBlock = (STableDataBlocks*)block; +int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { + STableDataCxt* pBlock = (STableDataCxt*)block; + int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); - if (keepBuf) { - taosMemoryFreeClear(pBlock->pData); - pBlock->pData = taosMemoryMalloc(TSDB_PAYLOAD_SIZE); - if (NULL == pBlock->pData) { - return TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < colNum; ++i) { + SColData* pCol = (SColData*)taosArrayGet(pBlock->pData->aCol, i); + if (deepClear) { + tColDataDeepClear(pCol); + } else { + tColDataClear(pCol); } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - } else { - pBlock->pData = NULL; } - pBlock->ordered = true; - pBlock->prevTS = INT64_MIN; - pBlock->size = sizeof(SSubmitBlk); - pBlock->tsSource = -1; - pBlock->numOfTables = 1; - pBlock->nAllocSize = TSDB_PAYLOAD_SIZE; - pBlock->headerSize = pBlock->size; - pBlock->createTbReqLen = 0; - - memset(&pBlock->rowBuilder, 0, sizeof(pBlock->rowBuilder)); - return TSDB_CODE_SUCCESS; } -int32_t qCloneStmtDataBlock(void** pDst, void* pSrc) { - *pDst = taosMemoryMalloc(sizeof(STableDataBlocks)); +int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset) { + int32_t code = 0; + + *pDst = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == *pDst) { return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(*pDst, pSrc, sizeof(STableDataBlocks)); - ((STableDataBlocks*)(*pDst))->cloned = true; + STableDataCxt* pNewCxt = (STableDataCxt*)*pDst; + STableDataCxt* pCxt = (STableDataCxt*)pSrc; + pNewCxt->pSchema = NULL; + pNewCxt->pValues = NULL; - STableDataBlocks* pBlock = (STableDataBlocks*)(*pDst); - if (pBlock->pTableMeta) { - void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pBlock->pTableMeta)); + if (pCxt->pMeta) { + void* pNewMeta = taosMemoryMalloc(TABLE_META_SIZE(pCxt->pMeta)); if (NULL == pNewMeta) { - taosMemoryFreeClear(*pDst); + insDestroyTableDataCxt(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pNewMeta, pBlock->pTableMeta, TABLE_META_SIZE(pBlock->pTableMeta)); - pBlock->pTableMeta = pNewMeta; + memcpy(pNewMeta, pCxt->pMeta, TABLE_META_SIZE(pCxt->pMeta)); + pNewCxt->pMeta = pNewMeta; } - if (pBlock->boundColumnInfo.boundColumns) { - int32_t size = pBlock->boundColumnInfo.numOfCols * sizeof(col_id_t); - void* tmp = taosMemoryMalloc(size); - memcpy(tmp, pBlock->boundColumnInfo.boundColumns, size); - pBlock->boundColumnInfo.boundColumns = tmp; - } + memcpy(&pNewCxt->boundColsInfo, &pCxt->boundColsInfo, sizeof(pCxt->boundColsInfo)); + pNewCxt->boundColsInfo.pColIndex = NULL; - if (pBlock->boundColumnInfo.cols) { - int32_t size = pBlock->boundColumnInfo.numOfCols * sizeof(SBoundColumn); - void* tmp = taosMemoryMalloc(size); - memcpy(tmp, pBlock->boundColumnInfo.cols, size); - pBlock->boundColumnInfo.cols = tmp; + if (pCxt->boundColsInfo.pColIndex) { + void* pNewColIdx = taosMemoryMalloc(pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + if (NULL == pNewColIdx) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(pNewColIdx, pCxt->boundColsInfo.pColIndex, + pCxt->boundColsInfo.numOfBound * sizeof(*pCxt->boundColsInfo.pColIndex)); + pNewCxt->boundColsInfo.pColIndex = pNewColIdx; } - if (pBlock->boundColumnInfo.colIdxInfo) { - int32_t size = pBlock->boundColumnInfo.numOfBound * sizeof(SBoundIdxInfo); - void* tmp = taosMemoryMalloc(size); - memcpy(tmp, pBlock->boundColumnInfo.colIdxInfo, size); - pBlock->boundColumnInfo.colIdxInfo = tmp; + if (pCxt->pData) { + SSubmitTbData* pNewTb = (SSubmitTbData*)taosMemoryMalloc(sizeof(SSubmitTbData)); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(pNewTb, pCxt->pData, sizeof(*pCxt->pData)); + pNewTb->pCreateTbReq = NULL; + + pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); + if (NULL == pNewTb) { + insDestroyTableDataCxt(*pDst); + return TSDB_CODE_OUT_OF_MEMORY; + } + + pNewCxt->pData = pNewTb; + + if (reset) { + code = qResetStmtDataBlock(*pDst, true); + } } - return qResetStmtDataBlock(*pDst, false); + return code; } -int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc, uint64_t uid, int32_t vgId) { - int32_t code = qCloneStmtDataBlock(pDst, pSrc); +int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, + bool rebuildCreateTb) { + int32_t code = qCloneStmtDataBlock(pDst, pSrc, false); if (code) { return code; } - STableDataBlocks* pBlock = (STableDataBlocks*)*pDst; - pBlock->pData = taosMemoryMalloc(pBlock->nAllocSize); - if (NULL == pBlock->pData) { - qDestroyStmtDataBlock(pBlock); - return TSDB_CODE_OUT_OF_MEMORY; + STableDataCxt* pBlock = (STableDataCxt*)*pDst; + if (pBlock->pMeta) { + pBlock->pMeta->uid = uid; + pBlock->pMeta->vgId = vgId; + pBlock->pMeta->suid = suid; } - pBlock->vgId = vgId; + pBlock->pData->suid = suid; + pBlock->pData->uid = uid; - if (pBlock->pTableMeta) { - pBlock->pTableMeta->uid = uid; - pBlock->pTableMeta->vgId = vgId; + if (rebuildCreateTb && NULL == pBlock->pData->pCreateTbReq) { + pBlock->pData->pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == pBlock->pData->pCreateTbReq) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - memset(pBlock->pData, 0, sizeof(SSubmitBlk)); - return TSDB_CODE_SUCCESS; } -STableMeta* qGetTableMetaInDataBlock(void* pDataBlock) { return ((STableDataBlocks*)pDataBlock)->pTableMeta; } +STableMeta* qGetTableMetaInDataBlock(STableDataCxt* pDataBlock) { return ((STableDataCxt*)pDataBlock)->pMeta; } -void qFreeStmtDataBlock(void* pDataBlock) { - if (pDataBlock == NULL) { - return; - } - - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pTableMeta); - taosMemoryFreeClear(((STableDataBlocks*)pDataBlock)->pData); - taosMemoryFreeClear(pDataBlock); -} - -void qDestroyStmtDataBlock(void* pBlock) { +void qDestroyStmtDataBlock(STableDataCxt* pBlock) { if (pBlock == NULL) { return; } - STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; - - pDataBlock->cloned = false; - insDestroyDataBlock(pDataBlock); + STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; + insDestroyTableDataCxt(pDataBlock); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 73cedfeb3d4109622062477214cd148bdd441391..52d3569dcd15be1370ee2930796e1bd126efc09a 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -20,938 +20,676 @@ #include "parUtil.h" #include "querynodes.h" #include "tRealloc.h" +#include "tdatablock.h" -typedef struct SBlockKeyTuple { - TSKEY skey; - void* payloadAddr; - int16_t index; -} SBlockKeyTuple; - -typedef struct SBlockKeyInfo { - int32_t maxBytesAlloc; - SBlockKeyTuple* pKeyTuple; -} SBlockKeyInfo; +void qDestroyBoundColInfo(void* pInfo) { + if (NULL == pInfo) { + return; + } -typedef struct { - int32_t index; - SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree) - STSchema* pSchema; - int64_t tbUid; // suid for child table, uid for normal table -} SBlockRowMerger; + SBoundColInfo* pBoundInfo = (SBoundColInfo*)pInfo; -static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - pMerger->index = -1; - } + taosMemoryFreeClear(pBoundInfo->pColIndex); } -static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) { - if (pMerger) { - int32_t size = taosArrayGetSize(pMerger->rowArray); - for (int32_t i = 0; i < size; ++i) { - tFree(*(void**)taosArrayGet(pMerger->rowArray, i)); +static char* tableNameGetPosition(SToken* pToken, char target) { + bool inEscape = false; + bool inQuote = false; + char quotaStr = 0; + + for (uint32_t i = 0; i < pToken->n; ++i) { + if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { + return pToken->z + i; } - taosArrayDestroy(pMerger->rowArray); - taosMemoryFreeClear(pMerger->pSchema); - taosMemoryFree(pMerger); - } -} + if (*(pToken->z + i) == TS_ESCAPE_CHAR) { + if (!inQuote) { + inEscape = !inEscape; + } + } -static int32_t rowDataCompar(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { + if (!inEscape) { + if (!inQuote) { + quotaStr = *(pToken->z + i); + inQuote = !inQuote; + } else if (quotaStr == *(pToken->z + i)) { + inQuote = !inQuote; + } + } + } } -} -static int32_t rowDataComparStable(const void* lhs, const void* rhs) { - TSKEY left = *(TSKEY*)lhs; - TSKEY right = *(TSKEY*)rhs; - if (left == right) { - return ((SBlockKeyTuple*)lhs)->index - ((SBlockKeyTuple*)rhs)->index; - } else { - return left > right ? 1 : -1; - } + return NULL; } -int32_t insGetExtendedRowSize(STableDataBlocks* pBlock) { - STableComInfo* pTableInfo = &pBlock->pTableMeta->tableInfo; - ASSERT(pBlock->rowSize == pTableInfo->rowSize); - return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen + - (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1); -} +int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { + const char* msg1 = "name too long"; + const char* msg2 = "invalid database name"; + const char* msg3 = "db is not specified"; + const char* msg4 = "invalid table name"; -void insGetSTSRowAppendInfo(uint8_t rowType, SParsedDataColInfo* spd, col_id_t idx, int32_t* toffset, - col_id_t* colIdx) { - col_id_t schemaIdx = 0; - if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundColumns[idx]; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart - *colIdx = schemaIdx; - } else { - *toffset = idx * sizeof(SKvRowIdx); // the offset of SKvRowIdx - *colIdx = idx; + int32_t code = TSDB_CODE_SUCCESS; + char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); + + if (p != NULL) { // db has been specified in sql string so we ignore current db path + int32_t dbLen = p - pTableName->z; + if (dbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg2); } - } else { - ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; - if (TD_IS_TP_ROW_T(rowType)) { - *toffset = (spd->cols + schemaIdx)->toffset; - *colIdx = schemaIdx; - } else { - *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SKvRowIdx); - *colIdx = (spd->colIdxInfo + idx)->finalIdx; + char name[TSDB_DB_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, dbLen); + int32_t actualDbLen = strdequote(name); + + code = tNameSetDbName(pName, acctId, name, actualDbLen); + if (code != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); } - } -} -int32_t insSetBlockInfo(SSubmitBlk* pBlocks, STableDataBlocks* dataBuf, int32_t numOfRows, SMsgBuf* pMsg) { - pBlocks->suid = (TSDB_NORMAL_TABLE == dataBuf->pTableMeta->tableType ? 0 : dataBuf->pTableMeta->suid); - pBlocks->uid = dataBuf->pTableMeta->uid; - pBlocks->sversion = dataBuf->pTableMeta->sversion; - pBlocks->schemaLen = dataBuf->createTbReqLen; + int32_t tbLen = pTableName->n - dbLen - 1; + if (tbLen <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } - if (pBlocks->numOfRows + numOfRows >= INT32_MAX) { - return buildInvalidOperationMsg(pMsg, "too many rows in sql, total number of rows should be less than INT32_MAX"); - } - pBlocks->numOfRows += numOfRows; - return TSDB_CODE_SUCCESS; -} + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(tbname, p + 1, tbLen); + /*tbLen = */ strdequote(tbname); + + code = tNameFromString(pName, tbname, T_NAME_TABLE); + if (code != 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } else { // get current DB name first, and then set it into path + if (pTableName->n >= TSDB_TABLE_NAME_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } -void insSetBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, col_id_t numOfCols) { - pColList->numOfCols = numOfCols; - pColList->numOfBound = numOfCols; - pColList->orderStatus = ORDER_STATUS_ORDERED; // default is ORDERED for non-bound mode - pColList->boundColumns = taosMemoryCalloc(pColList->numOfCols, sizeof(col_id_t)); - pColList->cols = taosMemoryCalloc(pColList->numOfCols, sizeof(SBoundColumn)); - pColList->colIdxInfo = NULL; - pColList->flen = 0; - pColList->allNullLen = 0; - - int32_t nVar = 0; - for (int32_t i = 0; i < pColList->numOfCols; ++i) { - uint8_t type = pSchema[i].type; - if (i > 0) { - pColList->cols[i].offset = pColList->cols[i - 1].offset + pSchema[i - 1].bytes; - pColList->cols[i].toffset = pColList->flen; - pColList->flen += TYPE_BYTES[type]; + char name[TSDB_TABLE_FNAME_LEN] = {0}; + strncpy(name, pTableName->z, pTableName->n); + strdequote(name); + + if (dbName == NULL) { + return buildInvalidOperationMsg(pMsgBuf, msg3); } - switch (type) { - case TSDB_DATA_TYPE_BINARY: - pColList->allNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES); - ++nVar; - break; - case TSDB_DATA_TYPE_NCHAR: - pColList->allNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE); - ++nVar; - break; - default: - break; + + code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); + if (code != TSDB_CODE_SUCCESS) { + code = buildInvalidOperationMsg(pMsgBuf, msg2); + return code; + } + + code = tNameFromString(pName, name, T_NAME_TABLE); + if (code != 0) { + code = buildInvalidOperationMsg(pMsgBuf, msg1); } - pColList->boundColumns[i] = i; } - pColList->allNullLen += pColList->flen; - pColList->boundNullLen = pColList->allNullLen; // default set allNullLen - pColList->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT)); -} -int32_t insSchemaIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)lhs; - uint16_t right = *(uint16_t*)rhs; + if (NULL != strchr(pName->tname, '.')) { + code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + } - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; + return code; +} + +int16_t insFindCol(SToken* pColname, int16_t start, int16_t end, SSchema* pSchema) { + while (start < end) { + if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { + return start; + } + ++start; } + return -1; } -int32_t insBoundIdxCompar(const void* lhs, const void* rhs) { - uint16_t left = *(uint16_t*)POINTER_SHIFT(lhs, sizeof(uint16_t)); - uint16_t right = *(uint16_t*)POINTER_SHIFT(rhs, sizeof(uint16_t)); +void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, + SArray* tagName, uint8_t tagNum, int32_t ttl) { + pTbReq->type = TD_CHILD_TABLE; + pTbReq->name = taosStrdup(tname); + pTbReq->ctb.suid = suid; + pTbReq->ctb.tagNum = tagNum; + if (sname) pTbReq->ctb.stbName = taosStrdup(sname); + pTbReq->ctb.pTag = (uint8_t*)pTag; + pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); + pTbReq->ttl = ttl; + pTbReq->commentLen = -1; + + return; +} - if (left == right) { - return 0; - } else { - return left > right ? 1 : -1; +static void initBoundCols(int32_t ncols, int16_t* pBoundCols) { + for (int32_t i = 0; i < ncols; ++i) { + pBoundCols[i] = i; } } -void destroyBoundColumnInfo(void* pBoundInfo) { - if (NULL == pBoundInfo) { - return; +static void initColValues(STableMeta* pTableMeta, SArray* pValues) { + SSchema* pSchemas = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i) { + SColVal val = COL_VAL_NONE(pSchemas[i].colId, pSchemas[i].type); + taosArrayPush(pValues, &val); } - - SParsedDataColInfo* pColList = (SParsedDataColInfo*)pBoundInfo; - - taosMemoryFreeClear(pColList->boundColumns); - taosMemoryFreeClear(pColList->cols); - taosMemoryFreeClear(pColList->colIdxInfo); } -static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, STableMeta* pTableMeta, - STableDataBlocks** dataBlocks) { - STableDataBlocks* dataBuf = (STableDataBlocks*)taosMemoryCalloc(1, sizeof(STableDataBlocks)); - if (dataBuf == NULL) { +int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { + pInfo->numOfCols = numOfBound; + pInfo->numOfBound = numOfBound; + pInfo->pColIndex = taosMemoryCalloc(numOfBound, sizeof(int16_t)); + if (NULL == pInfo->pColIndex) { return TSDB_CODE_OUT_OF_MEMORY; } + initBoundCols(numOfBound, pInfo->pColIndex); + return TSDB_CODE_SUCCESS; +} - dataBuf->nAllocSize = (uint32_t)defaultSize; - dataBuf->headerSize = startOffset; +void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { + // once the data block is disordered, we do NOT keep last timestamp any more + if (!pTableCxt->ordered) { + return; + } - // the header size will always be the startOffset value, reserved for the subumit block header - if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize * 2; + if (tsKey < pTableCxt->lastTs) { + pTableCxt->ordered = false; } - dataBuf->pData = taosMemoryMalloc(dataBuf->nAllocSize); - if (dataBuf->pData == NULL) { - taosMemoryFreeClear(dataBuf); - return TSDB_CODE_OUT_OF_MEMORY; + if (tsKey == pTableCxt->lastTs) { + pTableCxt->duplicateTs = true; } - memset(dataBuf->pData, 0, sizeof(SSubmitBlk)); - dataBuf->pTableMeta = tableMetaDup(pTableMeta); + pTableCxt->lastTs = tsKey; + return; +} - SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; - SSchema* pSchema = getTableColumnSchema(dataBuf->pTableMeta); - insSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); +void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; - dataBuf->vgId = dataBuf->pTableMeta->vgId; +static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, + bool colMode) { + STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); + if (NULL == pTableCxt) { + return TSDB_CODE_OUT_OF_MEMORY; + } - assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + int32_t code = TSDB_CODE_SUCCESS; - *dataBlocks = dataBuf; - return TSDB_CODE_SUCCESS; -} + pTableCxt->lastTs = 0; + pTableCxt->ordered = true; + pTableCxt->duplicateTs = false; -int32_t insBuildCreateTbMsg(STableDataBlocks* pBlocks, SVCreateTbReq* pCreateTbReq) { - SEncoder coder = {0}; - char* pBuf; - int32_t len; - - int32_t ret = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, ret); - if (pBlocks->nAllocSize - pBlocks->size < len) { - pBlocks->nAllocSize += len + pBlocks->rowSize; - char* pTmp = taosMemoryRealloc(pBlocks->pData, pBlocks->nAllocSize); - if (pTmp != NULL) { - pBlocks->pData = pTmp; - memset(pBlocks->pData + pBlocks->size, 0, pBlocks->nAllocSize - pBlocks->size); + pTableCxt->pMeta = tableMetaDup(pTableMeta); + if (NULL == pTableCxt->pMeta) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pSchema = + tBuildTSchema(getTableColumnSchema(pTableMeta), pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); + if (NULL == pTableCxt->pSchema) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = insInitBoundColsInfo(pTableMeta->tableInfo.numOfColumns, &pTableCxt->boundColsInfo); + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pValues = taosArrayInit(pTableMeta->tableInfo.numOfColumns, sizeof(SColVal)); + if (NULL == pTableCxt->pValues) { + code = TSDB_CODE_OUT_OF_MEMORY; } else { - pBlocks->nAllocSize -= len + pBlocks->rowSize; - return TSDB_CODE_OUT_OF_MEMORY; + initColValues(pTableMeta, pTableCxt->pValues); + } + } + if (TSDB_CODE_SUCCESS == code) { + pTableCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); + if (NULL == pTableCxt->pData) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pTableCxt->pData->flags = NULL != *pCreateTbReq ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; + pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->suid = pTableMeta->suid; + pTableCxt->pData->uid = pTableMeta->uid; + pTableCxt->pData->sver = pTableMeta->sversion; + pTableCxt->pData->pCreateTbReq = *pCreateTbReq; + *pCreateTbReq = NULL; + if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); + if (NULL == pTableCxt->pData->aCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + pTableCxt->pData->aRowP = taosArrayInit(128, POINTER_BYTES); + if (NULL == pTableCxt->pData->aRowP) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } } } - pBuf = pBlocks->pData + pBlocks->size; - - tEncoderInit(&coder, pBuf, len); - int32_t code = tEncodeSVCreateTbReq(&coder, pCreateTbReq); - tEncoderClear(&coder); - pBlocks->size += len; - pBlocks->createTbReqLen = len; + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pTableCxt; + qDebug("tableDataCxt created, uid:%" PRId64 ", vgId:%d", pTableMeta->uid, pTableMeta->vgId); + } else { + taosMemoryFree(pTableCxt); + } return code; } -void insDestroyDataBlock(STableDataBlocks* pDataBlock) { - if (pDataBlock == NULL) { - return; +static void resetColValues(SArray* pValues) { + int32_t num = taosArrayGetSize(pValues); + for (int32_t i = 0; i < num; ++i) { + SColVal* pVal = taosArrayGet(pValues, i); + pVal->flag = CV_FLAG_NONE; } +} - taosMemoryFreeClear(pDataBlock->pData); - taosMemoryFreeClear(pDataBlock->pTableMeta); - destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); - taosMemoryFreeClear(pDataBlock); +int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode) { + STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); + if (NULL != tmp) { + *pTableCxt = *tmp; + resetColValues((*pTableCxt)->pValues); + return TSDB_CODE_SUCCESS; + } + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode); + if (TSDB_CODE_SUCCESS == code) { + void* pData = *pTableCxt; // deal scan coverity + code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); + } + return code; } -int32_t insGetDataBlockFromList(SHashObj* pHashList, void* id, int32_t idLen, int32_t size, int32_t startOffset, - int32_t rowSize, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, - SArray* pBlockList, SVCreateTbReq* pCreateTbReq) { - *dataBlocks = NULL; - STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)id, idLen); - if (t1 != NULL) { - *dataBlocks = *t1; +static void destroyColVal(void* p) { + SColVal* pVal = p; + if (TSDB_DATA_TYPE_NCHAR == pVal->type) { + taosMemoryFree(pVal->value.pData); } +} - if (*dataBlocks == NULL) { - int32_t ret = createDataBlock((size_t)size, rowSize, startOffset, pTableMeta, dataBlocks); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } +void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { + if (NULL == pTableCxt) { + return; + } - if (NULL != pCreateTbReq && NULL != pCreateTbReq->ctb.pTag) { - ret = insBuildCreateTbMsg(*dataBlocks, pCreateTbReq); - if (ret != TSDB_CODE_SUCCESS) { - insDestroyDataBlock(*dataBlocks); - return ret; - } - } + taosMemoryFreeClear(pTableCxt->pMeta); + tDestroyTSchema(pTableCxt->pSchema); + insDestroyBoundColInfo(&pTableCxt->boundColsInfo); + taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); + if (pTableCxt->pData) { + tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pTableCxt->pData); + } + taosMemoryFree(pTableCxt); +} - // converting to 'const char*' is to handle coverity scan errors - taosHashPut(pHashList, (const char*)id, idLen, (const char*)dataBlocks, POINTER_BYTES); - if (pBlockList) { - taosArrayPush(pBlockList, dataBlocks); - } +void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt) { + return; } - return TSDB_CODE_SUCCESS; + tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + taosMemoryFree(pVgCxt->pData); + taosMemoryFree(pVgCxt); } -void insDestroyBlockArrayList(SArray* pDataBlockList) { - if (pDataBlockList == NULL) { +void insDestroyVgroupDataCxtList(SArray* pVgCxtList) { + if (NULL == pVgCxtList) { return; } - size_t size = taosArrayGetSize(pDataBlockList); + size_t size = taosArrayGetSize(pVgCxtList); for (int32_t i = 0; i < size; i++) { - void* p = taosArrayGetP(pDataBlockList, i); - insDestroyDataBlock(p); + void* p = taosArrayGetP(pVgCxtList, i); + insDestroyVgroupDataCxt(p); } - taosArrayDestroy(pDataBlockList); + taosArrayDestroy(pVgCxtList); } -void insDestroyBlockHashmap(SHashObj* pDataBlockHash) { - if (pDataBlockHash == NULL) { +void insDestroyVgroupDataCxtHashMap(SHashObj* pVgCxtHash) { + if (NULL == pVgCxtHash) { return; } - void** p1 = taosHashIterate(pDataBlockHash, NULL); - while (p1) { - STableDataBlocks* pBlocks = *p1; - insDestroyDataBlock(pBlocks); + void** p = taosHashIterate(pVgCxtHash, NULL); + while (p) { + insDestroyVgroupDataCxt(*(SVgroupDataCxt**)p); - p1 = taosHashIterate(pDataBlockHash, p1); + p = taosHashIterate(pVgCxtHash, p); } - taosHashCleanup(pDataBlockHash); + taosHashCleanup(pVgCxtHash); } -// data block is disordered, sort it in ascending order -static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - int16_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed yet. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +void insDestroyTableDataCxtHashMap(SHashObj* pTableCxtHash) { + if (NULL == pTableCxtHash) { + return; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + void** p = taosHashIterate(pTableCxtHash, NULL); + while (p) { + insDestroyTableDataCxt(*(STableDataCxt**)p); - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; + p = taosHashIterate(pTableCxtHash, p); } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - // todo. qsort is unstable, if timestamp is same, should get the last one - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - int32_t i = 0; - int32_t j = 1; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; - - if (ti == tj) { - ++j; - continue; - } + taosHashCleanup(pTableCxtHash); +} - int32_t nextPos = (++i); - if (nextPos != j) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + j, sizeof(SBlockKeyTuple)); - } - ++j; +static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCxt) { + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(128, sizeof(SSubmitTbData)); + if (NULL == pVgCxt->pData->aSubmitTbData) { + return TSDB_CODE_OUT_OF_MEMORY; } - - dataBuf->ordered = true; - pBlocks->numOfRows = i + 1; } + taosArrayPush(pVgCxt->pData->aSubmitTbData, pTableCxt->pData); + taosMemoryFreeClear(pTableCxt->pData); - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; + qDebug("add tableDataCxt uid:%" PRId64 " to vgId:%d", pTableCxt->pMeta->uid, pVgCxt->vgId); - return 0; -} - -static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) { - if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) { - ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray)); - return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index); - } - return NULL; + return TSDB_CODE_SUCCESS; } -static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows, - SBlockRowMerger** pBlkRowMerger, int32_t rowSize) { - ASSERT(nDupRows > 1); - SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1); - ASSERT(pStartKeyTp->skey == pEndKeyTp->skey); - - // TODO: optimization if end row is all normal -#if 0 - STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr; - if(isNormal(pEndRow)) { // set the end row if it is normal and return directly - pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr; - return TSDB_CODE_SUCCESS; - } -#endif - - if (!(*pBlkRowMerger)) { - (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger)); - if (!(*pBlkRowMerger)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->index = -1; - if (!(*pBlkRowMerger)->rowArray) { - (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*)); - if (!(*pBlkRowMerger)->rowArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - } - } - - if ((*pBlkRowMerger)->pSchema) { - if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } else { - if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) { - taosMemoryFreeClear((*pBlkRowMerger)->pSchema); - } - } +static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, + SVgroupDataCxt** pOutput) { + SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); + if (NULL == pVgCxt) { + return TSDB_CODE_OUT_OF_MEMORY; } - - if (!(*pBlkRowMerger)->pSchema) { - (*pBlkRowMerger)->pSchema = - tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); - - if (!(*pBlkRowMerger)->pSchema) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid; + pVgCxt->pData = taosMemoryCalloc(1, sizeof(SSubmitReq2)); + if (NULL == pVgCxt->pData) { + insDestroyVgroupDataCxt(pVgCxt); + return TSDB_CODE_OUT_OF_MEMORY; } - void* pDestRow = NULL; - ++((*pBlkRowMerger)->index); - if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { - void** pAlloc = (void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); - if (tRealloc((uint8_t**)pAlloc, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - pDestRow = *pAlloc; + pVgCxt->vgId = pTableCxt->pMeta->vgId; + int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); + if (TSDB_CODE_SUCCESS == code) { + taosArrayPush(pVgroupList, &pVgCxt); + *pOutput = pVgCxt; } else { - if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { - return TSDB_CODE_FAILED; - } - taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow); - } - - // merge rows to pDestRow - STSchema* pSchema = (*pBlkRowMerger)->pSchema; - SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); - for (int32_t i = 0; i < pSchema->numOfCols; ++i) { - SColVal colVal = {0}; - for (int32_t j = 0; j < nDupRows; ++j) { - tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); - if (!COL_VAL_IS_NONE(&colVal)) { - break; - } - } - taosArrayPush(pArray, &colVal); + insDestroyVgroupDataCxt(pVgCxt); } - if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) { - taosArrayDestroy(pArray); - return TSDB_CODE_FAILED; - } - - taosArrayDestroy(pArray); - return TSDB_CODE_SUCCESS; + return code; } -// data block is disordered, sort it in ascending order, and merge dup rows if exists -static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo, - SBlockRowMerger** ppBlkRowMerger) { - SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; - STableMeta* pTableMeta = dataBuf->pTableMeta; - int32_t nRows = pBlocks->numOfRows; - - // size is less than the total size, since duplicated rows may be removed. - - // allocate memory - size_t nAlloc = nRows * sizeof(SBlockKeyTuple); - if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { - char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; - pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; +int insColDataComp(const void* lp, const void* rp) { + SColData* pLeft = (SColData*)lp; + SColData* pRight = (SColData*)rp; + if (pLeft->cid < pRight->cid) { + return -1; + } else if (pLeft->cid > pRight->cid) { + return 1; } - memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); - tdResetSBlockRowMerger(*ppBlkRowMerger); - - int32_t extendedRowSize = insGetExtendedRowSize(dataBuf); - SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - char* pBlockData = pBlocks->data + pBlocks->schemaLen; - int32_t n = 0; - while (n < nRows) { - pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); - pBlkKeyTuple->payloadAddr = pBlockData; - pBlkKeyTuple->index = n; + return 0; +} - // next loop - pBlockData += extendedRowSize; - ++pBlkKeyTuple; - ++n; +int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); + if (NULL == pVgroupHash || NULL == pVgroupList) { + taosHashCleanup(pVgroupHash); + taosArrayDestroy(pVgroupList); + return TSDB_CODE_OUT_OF_MEMORY; } - if (!dataBuf->ordered) { - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - - taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); - - pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; - bool hasDup = false; - int32_t nextPos = 0; - int32_t i = 0; - int32_t j = 1; + int32_t code = TSDB_CODE_SUCCESS; + bool colFormat = false; - while (j < nRows) { - TSKEY ti = (pBlkKeyTuple + i)->skey; - TSKEY tj = (pBlkKeyTuple + j)->skey; + void* p = taosHashIterate(pTableHash, NULL); + if (p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + colFormat = (0 != (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT)); + } - if (ti == tj) { - ++j; + while (TSDB_CODE_SUCCESS == code && NULL != p) { + STableDataCxt* pTableCxt = *(STableDataCxt**)p; + if (colFormat) { + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); + if (pCol->nVal <= 0) { + p = taosHashIterate(pTableHash, p); continue; } - if ((j - i) > 1) { - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; - } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - if (!hasDup) { - hasDup = true; - } - i = j; - } else { - if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); - } - ++i; + if (pTableCxt->pData->pCreateTbReq) { + pTableCxt->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE; } - ++nextPos; - ++j; - } + taosArraySort(pTableCxt->pData->aCol, insColDataComp); - if ((j - i) > 1) { - ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey); - if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { - return TSDB_CODE_FAILED; + tColDataSortMerge(pTableCxt->pData->aCol); + } else { + if (!pTableCxt->ordered) { + tRowSort(pTableCxt->pData->aRowP); + } + if (!pTableCxt->ordered || pTableCxt->duplicateTs) { + code = tRowMerge(pTableCxt->pData->aRowP, pTableCxt->pSchema, 0); } - (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); - } else if (hasDup) { - memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); } - dataBuf->ordered = true; - pBlocks->numOfRows = nextPos + 1; - } - - dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; - dataBuf->prevTS = INT64_MIN; - - return TSDB_CODE_SUCCESS; -} - -// Erase the empty space reserved for binary data -static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple) { - // TODO: optimize this function, handle the case while binary is not presented - int32_t nonDataLen = sizeof(SSubmitBlk) + pTableDataBlock->createTbReqLen; - SSubmitBlk* pBlock = pDataBlock; - memcpy(pDataBlock, pTableDataBlock->pData, nonDataLen); - pDataBlock = (char*)pDataBlock + nonDataLen; - - pBlock->schemaLen = pTableDataBlock->createTbReqLen; - pBlock->dataLen = 0; - - int32_t numOfRows = pBlock->numOfRows; - for (int32_t i = 0; i < numOfRows; ++i) { - void* payload = (blkKeyTuple + i)->payloadAddr; - TDRowLenT rowTLen = TD_ROW_LEN((STSRow*)payload); - memcpy(pDataBlock, payload, rowTLen); - pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen); - pBlock->dataLen += rowTLen; - } - - return pBlock->dataLen + pBlock->schemaLen; -} - -int32_t insMergeTableDataBlocks(SHashObj* pHashObj, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); - int code = 0; - SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); - - STableDataBlocks** p = taosHashIterate(pHashObj, NULL); - STableDataBlocks* pOneTableBlock = *p; - SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock - SBlockRowMerger* pBlkRowMerger = NULL; - - while (pOneTableBlock) { - SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; - if (pBlocks->numOfRows > 0) { - STableDataBlocks* dataBuf = NULL; - pOneTableBlock->pTableMeta->vgId = pOneTableBlock->vgId; // for schemaless, restore origin vgId - int32_t ret = insGetDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, - sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, - pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return ret; - } - ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0); - // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int64_t destSize = dataBuf->size + pOneTableBlock->size + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta) + - pOneTableBlock->createTbReqLen; - - if (dataBuf->nAllocSize < destSize) { - dataBuf->nAllocSize = (uint32_t)(destSize * 1.5); - char* tmp = taosMemoryRealloc(dataBuf->pData, dataBuf->nAllocSize); - if (tmp != NULL) { - dataBuf->pData = tmp; - } else { // failed to allocate memory, free already allocated memory and return error code - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return TSDB_CODE_OUT_OF_MEMORY; - } + if (TSDB_CODE_SUCCESS == code) { + SVgroupDataCxt* pVgCxt = NULL; + int32_t vgId = pTableCxt->pMeta->vgId; + void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == p) { + code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + } else { + pVgCxt = *(SVgroupDataCxt**)p; } - - if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) { - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - insDestroyBlockArrayList(pVnodeDataBlockList); - taosMemoryFreeClear(dataBuf->pData); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - return code; + if (TSDB_CODE_SUCCESS == code) { + code = fillVgroupDataCxt(pTableCxt, pVgCxt); } - ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); - - // erase the empty space reserved for binary data - int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple); - - dataBuf->size += (finalLen + sizeof(SSubmitBlk)); - assert(dataBuf->size <= dataBuf->nAllocSize); - dataBuf->numOfTables += 1; } - - p = taosHashIterate(pHashObj, p); - if (p == NULL) { - break; + if (TSDB_CODE_SUCCESS == code) { + p = taosHashIterate(pTableHash, p); } + } - pOneTableBlock = *p; + taosHashCleanup(pVgroupHash); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pVgroupList; + } else { + insDestroyVgroupDataCxtList(pVgroupList); } - // free the table data blocks; - tdFreeSBlockRowMerger(pBlkRowMerger); - taosHashCleanup(pVnodeDataBlockHashList); - taosMemoryFreeClear(blkKeyInfo.pKeyTuple); - *pVgDataBlocks = pVnodeDataBlockList; - return TSDB_CODE_SUCCESS; + return code; } -int32_t insAllocateMemForSize(STableDataBlocks* pDataBlock, int32_t allSize) { - size_t remain = pDataBlock->nAllocSize - pDataBlock->size; - uint32_t nAllocSizeOld = pDataBlock->nAllocSize; - - // expand the allocated size - if (remain < allSize) { - pDataBlock->nAllocSize = (pDataBlock->size + allSize) * 1.5; - - char* tmp = taosMemoryRealloc(pDataBlock->pData, (size_t)pDataBlock->nAllocSize); - if (tmp != NULL) { - pDataBlock->pData = tmp; - memset(pDataBlock->pData + pDataBlock->size, 0, pDataBlock->nAllocSize - pDataBlock->size); - } else { - // do nothing, if allocate more memory failed - pDataBlock->nAllocSize = nAllocSizeOld; +static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uint32_t* pLen) { + int32_t code = TSDB_CODE_SUCCESS; + uint32_t len = 0; + void* pBuf = NULL; + tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + if (TSDB_CODE_SUCCESS == code) { + SEncoder encoder; + len += sizeof(SSubmitReq2Msg); + pBuf = taosMemoryMalloc(len); + if (NULL == pBuf) { return TSDB_CODE_OUT_OF_MEMORY; } + ((SSubmitReq2Msg*)pBuf)->header.vgId = htonl(vgId); + ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); + ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); + tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); + code = tEncodeSSubmitReq2(&encoder, pReq); + tEncoderClear(&encoder); } - return TSDB_CODE_SUCCESS; + if (TSDB_CODE_SUCCESS == code) { + *pData = pBuf; + *pLen = len; + } else { + taosMemoryFree(pBuf); + } + return code; } -int32_t insInitRowBuilder(SRowBuilder* pBuilder, int16_t schemaVer, SParsedDataColInfo* pColInfo) { - ASSERT(pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols)); - tdSRowInit(pBuilder, schemaVer); - tdSRowSetExtendedInfo(pBuilder, pColInfo->numOfCols, pColInfo->numOfBound, pColInfo->flen, pColInfo->allNullLen, - pColInfo->boundNullLen); - return TSDB_CODE_SUCCESS; +static void destroyVgDataBlocks(void* p) { + SVgDataBlocks* pVg = p; + taosMemoryFree(pVg->pData); + taosMemoryFree(pVg); } -static char* tableNameGetPosition(SToken* pToken, char target) { - bool inEscape = false; - bool inQuote = false; - char quotaStr = 0; - - for (uint32_t i = 0; i < pToken->n; ++i) { - if (*(pToken->z + i) == target && (!inEscape) && (!inQuote)) { - return pToken->z + i; - } - - if (*(pToken->z + i) == TS_ESCAPE_CHAR) { - if (!inQuote) { - inEscape = !inEscape; - } - } - - if (*(pToken->z + i) == '\'' || *(pToken->z + i) == '"') { - if (!inEscape) { - if (!inQuote) { - quotaStr = *(pToken->z + i); - inQuote = !inQuote; - } else if (quotaStr == *(pToken->z + i)) { - inQuote = !inQuote; - } - } - } +int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, SArray** pVgDataBlocks) { + size_t numOfVg = taosArrayGetSize(pVgDataCxtList); + SArray* pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); + if (NULL == pDataBlocks) { + return TSDB_CODE_OUT_OF_MEMORY; } - return NULL; -} - -int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const char* dbName, SMsgBuf* pMsgBuf) { - const char* msg1 = "name too long"; - const char* msg2 = "invalid database name"; - const char* msg3 = "db is not specified"; - const char* msg4 = "invalid table name"; - int32_t code = TSDB_CODE_SUCCESS; - char* p = tableNameGetPosition(pTableName, TS_PATH_DELIMITER[0]); - - if (p != NULL) { // db has been specified in sql string so we ignore current db path - assert(*p == TS_PATH_DELIMITER[0]); - - int32_t dbLen = p - pTableName->z; - if (dbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg2); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfVg; ++i) { + SVgroupDataCxt* src = taosArrayGetP(pVgDataCxtList, i); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + if (NULL == dst) { + code = TSDB_CODE_OUT_OF_MEMORY; } - char name[TSDB_DB_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, dbLen); - int32_t actualDbLen = strdequote(name); - - code = tNameSetDbName(pName, acctId, name, actualDbLen); - if (code != TSDB_CODE_SUCCESS) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); + code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); } - - int32_t tbLen = pTableName->n - dbLen - 1; - if (tbLen <= 0) { - return buildInvalidOperationMsg(pMsgBuf, msg4); - } - - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(tbname, p + 1, tbLen); - /*tbLen = */ strdequote(tbname); - - code = tNameFromString(pName, tbname, T_NAME_TABLE); - if (code != 0) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); } - } else { // get current DB name first, and then set it into path - if (pTableName->n >= TSDB_TABLE_NAME_LEN) { - return buildInvalidOperationMsg(pMsgBuf, msg1); + if (TSDB_CODE_SUCCESS == code) { + code = (NULL == taosArrayPush(pDataBlocks, &dst) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); } + } - assert(pTableName->n < TSDB_TABLE_FNAME_LEN); + if (TSDB_CODE_SUCCESS == code) { + *pVgDataBlocks = pDataBlocks; + } else { + taosArrayDestroyP(pDataBlocks, destroyVgDataBlocks); + } - char name[TSDB_TABLE_FNAME_LEN] = {0}; - strncpy(name, pTableName->z, pTableName->n); - strdequote(name); + return code; +} - if (dbName == NULL) { - return buildInvalidOperationMsg(pMsgBuf, msg3); - } +static int bindFileds(SBoundColInfo* pBoundInfo, SSchema* pSchema, TAOS_FIELD* fields, int numFields) { + bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool)); + if (NULL == pUseCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); - if (code != TSDB_CODE_SUCCESS) { - code = buildInvalidOperationMsg(pMsgBuf, msg2); - return code; - } + pBoundInfo->numOfBound = 0; - code = tNameFromString(pName, name, T_NAME_TABLE); - if (code != 0) { - code = buildInvalidOperationMsg(pMsgBuf, msg1); + int16_t lastColIdx = -1; // last column found + int32_t code = TSDB_CODE_SUCCESS; + for (int i = 0; i < numFields; i++) { + SToken token; + token.z = fields[i].name; + token.n = strlen(fields[i].name); + + int16_t t = lastColIdx + 1; + int16_t index = insFindCol(&token, t, pBoundInfo->numOfCols, pSchema); + if (index < 0 && t > 0) { + index = insFindCol(&token, 0, t, pSchema); + } + if (index < 0) { + uError("can not find column name:%s", token.z); + code = TSDB_CODE_PAR_INVALID_COLUMN; + break; + } else if (pUseCols[index]) { + code = TSDB_CODE_PAR_INVALID_COLUMN; + uError("duplicated column name:%s", token.z); + break; + } else { + lastColIdx = index; + pUseCols[index] = true; + pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index; + ++pBoundInfo->numOfBound; } } - if (NULL != strchr(pName->tname, '.')) { - code = generateSyntaxErrMsgExt(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, "The table name cannot contain '.'"); + if (TSDB_CODE_SUCCESS == code && !pUseCols[0]) { + uError("primary timestamp column can not be null:"); + code = TSDB_CODE_PAR_INVALID_COLUMN; } + taosMemoryFree(pUseCols); return code; } -int32_t insFindCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) { - while (start < end) { - if (strlen(pSchema[start].name) == pColname->n && strncmp(pColname->z, pSchema[start].name, pColname->n) == 0) { - return start; +int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, TAOS_FIELD* tFields, + int numFields, bool needChangeLength) { + STableDataCxt* pTableCxt = NULL; + int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, + sizeof(pTableMeta->uid), pTableMeta, &pCreateTb, &pTableCxt, true); + if (ret != TSDB_CODE_SUCCESS) { + uError("insGetTableDataCxt error"); + goto end; + } + if (tFields != NULL) { + ret = bindFileds(&pTableCxt->boundColsInfo, getTableColumnSchema(pTableMeta), tFields, numFields); + if (ret != TSDB_CODE_SUCCESS) { + uError("bindFileds error"); + goto end; } - ++start; } - return -1; -} + // no need to bind, because select * get all fields + ret = initTableColSubmitData(pTableCxt); + if (ret != TSDB_CODE_SUCCESS) { + uError("initTableColSubmitData error"); + goto end; + } -void insBuildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid, const char* sname, - SArray* tagName, uint8_t tagNum, int32_t ttl) { - pTbReq->type = TD_CHILD_TABLE; - pTbReq->name = strdup(tname); - pTbReq->ctb.suid = suid; - pTbReq->ctb.tagNum = tagNum; - if (sname) pTbReq->ctb.stbName = strdup(sname); - pTbReq->ctb.pTag = (uint8_t*)pTag; - pTbReq->ctb.tagName = taosArrayDup(tagName, NULL); - pTbReq->ttl = ttl; - pTbReq->commentLen = -1; + char* p = (char*)data; + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | + p += sizeof(int32_t); + p += sizeof(int32_t); - return; -} + int32_t numOfRows = *(int32_t*)p; + p += sizeof(int32_t); -int32_t insMemRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SMemParam* pa = (SMemParam*)param; - SRowBuilder* rb = pa->rb; + int32_t numOfCols = *(int32_t*)p; + p += sizeof(int32_t); - if (value == NULL) { // it is a null data - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, pa->colIdx); - return TSDB_CODE_SUCCESS; - } + p += sizeof(int32_t); + p += sizeof(uint64_t); - if (TSDB_DATA_TYPE_BINARY == pa->schema->type) { - const char* rowEnd = tdRowEnd(rb->pBuf); - STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - int32_t output = 0; - const char* rowEnd = tdRowEnd(rb->pBuf); - if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), "%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - varDataSetLen(rowEnd, output); - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx); - } else { - tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, false, pa->toffset, pa->colIdx); - } + int8_t* fields = p; + p += numOfCols * (sizeof(int8_t) + sizeof(int32_t)); - return TSDB_CODE_SUCCESS; -} + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; -int32_t insCheckTimestamp(STableDataBlocks* pDataBlocks, const char* start) { - // once the data block is disordered, we do NOT keep previous timestamp any more - if (!pDataBlocks->ordered) { - return TSDB_CODE_SUCCESS; - } + char* pStart = p; - TSKEY k = *(TSKEY*)start; - if (k <= pDataBlocks->prevTS) { - pDataBlocks->ordered = false; + SSchema* pSchema = getTableColumnSchema(pTableCxt->pMeta); + SBoundColInfo* boundInfo = &pTableCxt->boundColsInfo; + + if (boundInfo->numOfBound != numOfCols) { + uError("boundInfo->numOfBound:%d != numOfCols:%d", boundInfo->numOfBound, numOfCols); + ret = TSDB_CODE_INVALID_PARA; + goto end; } + for (int c = 0; c < boundInfo->numOfBound; ++c) { + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); - pDataBlocks->prevTS = k; - return TSDB_CODE_SUCCESS; -} + if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + uError("type or bytes not equal"); + ret = TSDB_CODE_INVALID_PARA; + goto end; + } -static void buildMsgHeader(STableDataBlocks* src, SVgDataBlocks* blocks) { - SSubmitReq* submit = (SSubmitReq*)blocks->pData; - submit->header.vgId = htonl(blocks->vg.vgId); - submit->header.contLen = htonl(blocks->size); - submit->length = submit->header.contLen; - submit->numOfBlocks = htonl(blocks->numOfTables); - SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); - int32_t numOfBlocks = blocks->numOfTables; - while (numOfBlocks--) { - int32_t dataLen = blk->dataLen; - int32_t schemaLen = blk->schemaLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htonl(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + schemaLen + dataLen); - } -} + int8_t* offset = pStart; + if (IS_VAR_DATA_TYPE(pColSchema->type)) { + pStart += numOfRows * sizeof(int32_t); + } else { + pStart += BitmapLen(numOfRows); + } + char* pData = pStart; -int32_t insBuildOutput(SHashObj* pVgroupsHashObj, SArray* pVgDataBlocks, SArray** pDataBlocks) { - size_t numOfVg = taosArrayGetSize(pVgDataBlocks); - *pDataBlocks = taosArrayInit(numOfVg, POINTER_BYTES); - if (NULL == *pDataBlocks) { - return TSDB_CODE_OUT_OF_MEMORY; - } - for (size_t i = 0; i < numOfVg; ++i) { - STableDataBlocks* src = taosArrayGetP(pVgDataBlocks, i); - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); - if (NULL == dst) { - return TSDB_CODE_OUT_OF_MEMORY; + tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); + fields += sizeof(int8_t) + sizeof(int32_t); + if (needChangeLength) { + pStart += htonl(colLength[c]); + } else { + pStart += colLength[c]; } - taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - dst->numOfTables = src->numOfTables; - dst->size = src->size; - TSWAP(dst->pData, src->pData); - buildMsgHeader(src, dst); - taosArrayPush(*pDataBlocks, &dst); } - return TSDB_CODE_SUCCESS; + +end: + return ret; } diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index b248efee59584156bace88e5dd0f001003216bc0..678dcf34d3a9a1d08bfa8c51e25abe3cd2161daf 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -61,6 +61,7 @@ static SKeyword keywordTable[] = { {"COLUMN", TK_COLUMN}, {"COMMENT", TK_COMMENT}, {"COMP", TK_COMP}, + {"COMPACT", TK_COMPACT}, {"CONNECTION", TK_CONNECTION}, {"CONNECTIONS", TK_CONNECTIONS}, {"CONNS", TK_CONNS}, @@ -90,6 +91,7 @@ static SKeyword keywordTable[] = { {"EXISTS", TK_EXISTS}, {"EXPIRED", TK_EXPIRED}, {"EXPLAIN", TK_EXPLAIN}, + {"EVENT_WINDOW", TK_EVENT_WINDOW}, {"EVERY", TK_EVERY}, {"FILE", TK_FILE}, {"FILL", TK_FILL}, @@ -196,15 +198,16 @@ static SKeyword keywordTable[] = { {"SNODES", TK_SNODES}, {"SOFFSET", TK_SOFFSET}, {"SPLIT", TK_SPLIT}, - {"STT_TRIGGER", TK_STT_TRIGGER}, {"STABLE", TK_STABLE}, {"STABLES", TK_STABLES}, + {"START", TK_START}, {"STATE", TK_STATE}, {"STATE_WINDOW", TK_STATE_WINDOW}, {"STORAGE", TK_STORAGE}, {"STREAM", TK_STREAM}, {"STREAMS", TK_STREAMS}, {"STRICT", TK_STRICT}, + {"STT_TRIGGER", TK_STT_TRIGGER}, {"SUBSCRIBE", TK_SUBSCRIBE}, {"SUBSCRIPTIONS", TK_SUBSCRIPTIONS}, {"SUBTABLE", TK_SUBTABLE}, @@ -261,6 +264,7 @@ static SKeyword keywordTable[] = { {"WRITE", TK_WRITE}, {"_C0", TK_ROWTS}, {"_IROWTS", TK_IROWTS}, + {"_ISFILLED", TK_ISFILLED}, {"_QDURATION", TK_QDURATION}, {"_QEND", TK_QEND}, {"_QSTART", TK_QSTART}, @@ -269,6 +273,7 @@ static SKeyword keywordTable[] = { {"_WDURATION", TK_WDURATION}, {"_WEND", TK_WEND}, {"_WSTART", TK_WSTART}, + {"ALIVE", TK_ALIVE}, }; // clang-format on @@ -718,14 +723,3 @@ void taosCleanupKeywordsTable() { taosHashCleanup(m); } } - -SToken taosTokenDup(SToken* pToken, char* buf, int32_t len) { - assert(pToken != NULL && buf != NULL && len > pToken->n); - - strncpy(buf, pToken->z, pToken->n); - buf[pToken->n] = 0; - - SToken token = *pToken; - token.z = buf; - return token; -} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8b3cfd105b57ce101a03c5e4879756b02f8012b4..8c5b41a7ee3e9b20e48454ab1d16ae99ebfc401d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -352,7 +352,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta); } } - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, tstrerror(code), pName->dbname, pName->tname); } @@ -787,7 +787,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->tableType = pTable->pMeta->tableType; pCol->colId = pColSchema->colId; pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN); - pCol->hasIndex = (0 == tagFlag); + pCol->hasIndex = ((0 == tagFlag) || (pColSchema != NULL && IS_IDX_ON(pColSchema))); pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { @@ -1470,8 +1470,8 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc SNode* pTable = pSelect->pFromTable; if ((NULL != pTable && (QUERY_NODE_REAL_TABLE != nodeType(pTable) || - (TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && - TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType)))) { + (TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && + TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType)))) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, "%s is only supported in single table query", pFunc->functionName); } @@ -1692,7 +1692,7 @@ static int32_t rewriteFuncToValue(STranslateContext* pCxt, char* pLiteral, SNode static int32_t rewriteDatabaseFunc(STranslateContext* pCxt, SNode** pNode) { char* pCurrDb = NULL; if (NULL != pCxt->pParseCxt->db) { - pCurrDb = taosMemoryStrDup((void*)pCxt->pParseCxt->db); + pCurrDb = taosStrdup((void*)pCxt->pParseCxt->db); if (NULL == pCurrDb) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1701,7 +1701,7 @@ static int32_t rewriteDatabaseFunc(STranslateContext* pCxt, SNode** pNode) { } static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) { - char* pVer = taosMemoryStrDup((void*)version); + char* pVer = taosStrdup((void*)version); if (NULL == pVer) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1709,7 +1709,7 @@ static int32_t rewriteClentVersionFunc(STranslateContext* pCxt, SNode** pNode) { } static int32_t rewriteServerVersionFunc(STranslateContext* pCxt, SNode** pNode) { - char* pVer = taosMemoryStrDup((void*)pCxt->pParseCxt->svrVer); + char* pVer = taosStrdup((void*)pCxt->pParseCxt->svrVer); if (NULL == pVer) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1720,7 +1720,7 @@ static int32_t rewriteServerStatusFunc(STranslateContext* pCxt, SNode** pNode) { if (pCxt->pParseCxt->nodeOffline) { return TSDB_CODE_RPC_NETWORK_UNAVAIL; } - char* pStatus = taosMemoryStrDup((void*)"1"); + char* pStatus = taosStrdup((void*)"1"); return rewriteFuncToValue(pCxt, pStatus, pNode); } @@ -1728,7 +1728,7 @@ static int32_t rewriteUserFunc(STranslateContext* pCxt, SNode** pNode) { char userConn[TSDB_USER_LEN + 1 + TSDB_FQDN_LEN] = {0}; // format 'user@host' int32_t len = snprintf(userConn, sizeof(userConn), "%s@", pCxt->pParseCxt->pUser); taosGetFqdn(userConn + len); - char* pUserConn = taosMemoryStrDup((void*)userConn); + char* pUserConn = taosStrdup((void*)userConn); if (NULL == pUserConn) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2250,7 +2250,8 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { } static bool sysTableFromVnode(const char* pTable) { - return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS))); + return ((0 == strcmp(pTable, TSDB_INS_TABLE_TABLES)) || (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) || + (0 == strcmp(pTable, TSDB_INS_TABLE_COLS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -2318,7 +2319,9 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, ((SSelectStmt*)pCxt->pCurrStmt)->isEmptyResult = true; } - if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) { + if (TSDB_CODE_SUCCESS == code && + (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && !hasUserDbCond) || + 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &pVgs); } @@ -2416,7 +2419,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_TAGS) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_COLS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -2992,11 +2996,14 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* return TSDB_CODE_SUCCESS; } - if (!pCxt->createStream && (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER) || - TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER))) { + if (!pCxt->createStream && TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_INITIALIZER)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE); } + if (TSWINDOW_IS_EQUAL(pFill->timeRange, TSWINDOW_DESC_INITIALIZER)) { + return TSDB_CODE_SUCCESS; + } + // interp FILL clause if (NULL == pInterval) { return TSDB_CODE_SUCCESS; @@ -3188,6 +3195,15 @@ static int32_t translateSessionWindow(STranslateContext* pCxt, SSelectStmt* pSel return TSDB_CODE_SUCCESS; } +static int32_t translateEventWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && + !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TIMELINE_QUERY, + "EVENT_WINDOW requires valid time series input"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: @@ -3196,6 +3212,8 @@ static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSe return translateSessionWindow(pCxt, pSelect); case QUERY_NODE_INTERVAL_WINDOW: return translateIntervalWindow(pCxt, pSelect); + case QUERY_NODE_EVENT_WINDOW: + return translateEventWindow(pCxt, pSelect); default: break; } @@ -3307,9 +3325,6 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { } static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL == pSelect->pPartitionByList) { - return TSDB_CODE_SUCCESS; - } pCxt->currClause = SQL_CLAUSE_PARTITION_BY; int32_t code = translateExprList(pCxt, pSelect->pPartitionByList); if (TSDB_CODE_SUCCESS == code) { @@ -3543,7 +3558,7 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS if (comp > 0) { SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code || NULL == pRightFunc) { // deal scan coverity return code; } REPLACE_LIST2_NODE(pRightFunc); @@ -3551,7 +3566,7 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS } else if (comp < 0) { SNode* pLeftFunc = NULL; int32_t code = createCastFunc(pCxt, pLeft, pRightExpr->resType, &pLeftFunc); - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != code || NULL == pLeftFunc) { // deal scan coverity return code; } REPLACE_LIST1_NODE(pLeftFunc); @@ -3932,7 +3947,8 @@ static int32_t checkDbKeepOption(STranslateContext* pCxt, SDatabaseOptions* pOpt if (pOptions->keep[0] < TSDB_MIN_KEEP || pOptions->keep[1] < TSDB_MIN_KEEP || pOptions->keep[2] < TSDB_MIN_KEEP || pOptions->keep[0] > tsdbMaxKeep || pOptions->keep[1] > tsdbMaxKeep || pOptions->keep[2] > tsdbMaxKeep) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, - "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 " valid range: [%dm, %dm]", + "Invalid option keep: %" PRId64 ", %" PRId64 ", %" PRId64 + " valid range: [%dm, %" PRId64 "m]", pOptions->keep[0], pOptions->keep[1], pOptions->keep[2], TSDB_MIN_KEEP, tsdbMaxKeep); } @@ -4599,58 +4615,109 @@ typedef struct SSampleAstInfo { SNode* pSliding; SNodeList* pPartitionByList; STableMeta* pRollupTableMeta; + bool createSmaIndex; } SSampleAstInfo; -static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, - int32_t* pExprLen) { - SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); - if (NULL == pSelect) { - return TSDB_CODE_OUT_OF_MEMORY; - } - sprintf(pSelect->stmtName, "%p", pSelect); - +static int32_t buildTableForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SRealTableNode* pTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); if (NULL == pTable) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } snprintf(pTable->table.dbName, sizeof(pTable->table.dbName), "%s", pInfo->pDbName); snprintf(pTable->table.tableName, sizeof(pTable->table.tableName), "%s", pInfo->pTableName); TSWAP(pTable->pMeta, pInfo->pRollupTableMeta); - pSelect->pFromTable = (SNode*)pTable; + *pOutput = (SNode*)pTable; + return TSDB_CODE_SUCCESS; +} - TSWAP(pSelect->pProjectionList, pInfo->pFuncs); +static int32_t addWstartToSampleProjects(SNodeList* pProjectionList) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - if (NULL == pSelect->pProjectionList || NULL == pFunc) { - nodesDestroyNode((SNode*)pSelect); + if (NULL == pFunc) { return TSDB_CODE_OUT_OF_MEMORY; } strcpy(pFunc->functionName, "_wstart"); - nodesListPushFront(pSelect->pProjectionList, (SNode*)pFunc); - SNode* pProject = NULL; - FOREACH(pProject, pSelect->pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + return nodesListPushFront(pProjectionList, (SNode*)pFunc); +} + +static int32_t addWendToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wend"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} + +static int32_t addWdurationToSampleProjects(SNodeList* pProjectionList) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "_wduration"); + return nodesListAppend(pProjectionList, (SNode*)pFunc); +} + +static int32_t buildProjectsForSampleAst(SSampleAstInfo* pInfo, SNodeList** pList) { + SNodeList* pProjectionList = pInfo->pFuncs; + pInfo->pFuncs = NULL; + + int32_t code = addWstartToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code && pInfo->createSmaIndex) { + code = addWendToSampleProjects(pProjectionList); + if (TSDB_CODE_SUCCESS == code) { + code = addWdurationToSampleProjects(pProjectionList); + } + } - TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); + if (TSDB_CODE_SUCCESS == code) { + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { sprintf(((SExprNode*)pProject)->aliasName, "#%p", pProject); } + *pList = pProjectionList; + } else { + nodesDestroyList(pProjectionList); + } + return code; +} +static int32_t buildIntervalForSampleAst(SSampleAstInfo* pInfo, SNode** pOutput) { SIntervalWindowNode* pInterval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); if (NULL == pInterval) { - nodesDestroyNode((SNode*)pSelect); return TSDB_CODE_OUT_OF_MEMORY; } - pSelect->pWindow = (SNode*)pInterval; TSWAP(pInterval->pInterval, pInfo->pInterval); TSWAP(pInterval->pOffset, pInfo->pOffset); TSWAP(pInterval->pSliding, pInfo->pSliding); pInterval->pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pInterval->pCol) { - nodesDestroyNode((SNode*)pSelect); + nodesDestroyNode((SNode*)pInterval); return TSDB_CODE_OUT_OF_MEMORY; } ((SColumnNode*)pInterval->pCol)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; strcpy(((SColumnNode*)pInterval->pCol)->colName, ROWTS_PSEUDO_COLUMN_NAME); + *pOutput = (SNode*)pInterval; + return TSDB_CODE_SUCCESS; +} - pCxt->createStream = true; - int32_t code = translateQuery(pCxt, (SNode*)pSelect); +static int32_t buildSampleAst(STranslateContext* pCxt, SSampleAstInfo* pInfo, char** pAst, int32_t* pLen, char** pExpr, + int32_t* pExprLen) { + SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + if (NULL == pSelect) { + return TSDB_CODE_OUT_OF_MEMORY; + } + sprintf(pSelect->stmtName, "%p", pSelect); + + int32_t code = buildTableForSampleAst(pInfo, &pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = buildProjectsForSampleAst(pInfo, &pSelect->pProjectionList); + } + if (TSDB_CODE_SUCCESS == code) { + TSWAP(pSelect->pPartitionByList, pInfo->pPartitionByList); + code = buildIntervalForSampleAst(pInfo, &pSelect->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + pCxt->createStream = true; + code = translateQuery(pCxt, (SNode*)pSelect); + } if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString((SNode*)pSelect, false, pAst, pLen); } @@ -4843,7 +4910,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); pReq->numOfTags = LIST_LENGTH(pStmt->pTags); if (pStmt->pOptions->commentNull == false) { - pReq->pComment = strdup(pStmt->pOptions->comment); + pReq->pComment = taosStrdup(pStmt->pOptions->comment); if (NULL == pReq->pComment) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4911,7 +4978,7 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* if (TSDB_ALTER_TABLE_UPDATE_OPTIONS == pStmt->alterType) { // pAlterReq->ttl = pStmt->pOptions->ttl; if (pStmt->pOptions->commentNull == false) { - pAlterReq->comment = strdup(pStmt->pOptions->comment); + pAlterReq->comment = taosStrdup(pStmt->pOptions->comment); if (NULL == pAlterReq->comment) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -4958,10 +5025,10 @@ static int32_t buildAlterSuperTableReq(STranslateContext* pCxt, SAlterTableStmt* return TSDB_CODE_SUCCESS; } -static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) { +static const SSchema* getColSchema(const STableMeta* pTableMeta, const char* pColName) { int32_t numOfFields = getNumOfTags(pTableMeta) + getNumOfColumns(pTableMeta); for (int32_t i = 0; i < numOfFields; ++i) { - SSchema* pSchema = pTableMeta->schema + i; + const SSchema* pSchema = pTableMeta->schema + i; if (0 == strcmp(pColName, pSchema->name)) { return pSchema; } @@ -4969,7 +5036,19 @@ static SSchema* getColSchema(STableMeta* pTableMeta, const char* pColName) { return NULL; } -static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) { +static const SSchema* getNormalColSchema(const STableMeta* pTableMeta, const char* pColName) { + int32_t numOfCols = getNumOfColumns(pTableMeta); + SSchema* pColsSchema = getTableColumnSchema(pTableMeta); + for (int32_t i = 0; i < numOfCols; ++i) { + const SSchema* pSchema = pColsSchema + i; + if (0 == strcmp(pColName, pSchema->name)) { + return pSchema; + } + } + return NULL; +} + +static SSchema* getTagSchema(const STableMeta* pTableMeta, const char* pTagName) { int32_t numOfTags = getNumOfTags(pTableMeta); SSchema* pTagsSchema = getTableTagSchema(pTableMeta); for (int32_t i = 0; i < numOfTags; ++i) { @@ -4981,7 +5060,8 @@ static SSchema* getTagSchema(STableMeta* pTableMeta, const char* pTagName) { return NULL; } -static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta) { +static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTableStmt* pStmt, + const STableMeta* pTableMeta) { SSchema* pTagsSchema = getTableTagSchema(pTableMeta); if (getNumOfTags(pTableMeta) == 1 && pTagsSchema->type == TSDB_DATA_TYPE_JSON && (pStmt->alterType == TSDB_ALTER_TABLE_ADD_TAG || pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG || @@ -5004,7 +5084,7 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -5127,7 +5207,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { SCreateUserReq createReq = {0}; - strcpy(createReq.user, pStmt->useName); + strcpy(createReq.user, pStmt->userName); createReq.createType = 0; createReq.superUser = 0; createReq.sysInfo = pStmt->sysinfo; @@ -5139,7 +5219,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { SAlterUserReq alterReq = {0}; - strcpy(alterReq.user, pStmt->useName); + strcpy(alterReq.user, pStmt->userName); alterReq.alterType = pStmt->alterType; alterReq.superUser = 0; alterReq.enable = pStmt->enable; @@ -5154,7 +5234,7 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt static int32_t translateDropUser(STranslateContext* pCxt, SDropUserStmt* pStmt) { SDropUserReq dropReq = {0}; - strcpy(dropReq.user, pStmt->useName); + strcpy(dropReq.user, pStmt->userName); return buildCmdMsg(pCxt, TDMT_MND_DROP_USER, (FSerializeFunc)tSerializeSDropUserReq, &dropReq); } @@ -5197,7 +5277,7 @@ static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, } static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLen) { - *pSql = strdup(pCxt->pParseCxt->pSql); + *pSql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == *pSql) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -5206,6 +5286,7 @@ static int32_t getSmaIndexSql(STranslateContext* pCxt, char** pSql, int32_t* pLe } static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SSampleAstInfo* pInfo) { + pInfo->createSmaIndex = true; pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = nodesCloneList(pStmt->pOptions->pFuncs); @@ -5320,6 +5401,28 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + memset(&name, 0, sizeof(SName)); + + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + memset(&name, 0, sizeof(SName)); + + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, pReq->dbFName); + + SNode* pNode = NULL; + ASSERT(LIST_LENGTH(pStmt->pCols) == 1); + FOREACH(pNode, pStmt->pCols) { + SColumnNode* p = (SColumnNode*)pNode; + memcpy(pReq->colName, p->colName, sizeof(p->colName)); + } + + // impl later + return TSDB_CODE_SUCCESS; +} + static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { SMCreateFullTextReq createFTReq = {0}; int32_t code = buildCreateFullTextReq(pCxt, pStmt, &createFTReq); @@ -5330,9 +5433,20 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde return code; } +static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SCreateTagIndexReq createTagIdxReq = {0}; + int32_t code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); + } + return code; +} + static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { if (INDEX_TYPE_FULLTEXT == pStmt->indexType) { return translateCreateFullTextIndex(pCxt, pStmt); + } else if (INDEX_TYPE_NORMAL == pStmt->indexType) { + return translateCreateNormalIndex(pCxt, pStmt); } return translateCreateSmaIndex(pCxt, pStmt); } @@ -5401,7 +5515,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS pReq->igExists = pStmt->ignoreExists; pReq->withMeta = pStmt->withMeta; - pReq->sql = strdup(pCxt->pParseCxt->pSql); + pReq->sql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -5489,7 +5603,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt } static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { - // todo + // The statement is executed directly on the client without constructing a message. return TSDB_CODE_SUCCESS; } @@ -5504,6 +5618,14 @@ static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta); } +static int32_t translateCompact(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt) { + SCompactDbReq compactReq = {0}; + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, compactReq.db); + return buildCmdMsg(pCxt, TDMT_MND_COMPACT_DB, (FSerializeFunc)tSerializeSCompactDbReq, &compactReq); +} + static int32_t translateKillConnection(STranslateContext* pCxt, SKillStmt* pStmt) { SKillConnReq killReq = {0}; killReq.connId = pStmt->targetId; @@ -5602,6 +5724,13 @@ static int32_t addWstartTsToCreateStreamQuery(STranslateContext* pCxt, SNode* pS return code; } +static const char* getTagNameForCreateStreamTag(SNode* pTag) { + if (QUERY_NODE_COLUMN_DEF == nodeType(pTag)) { + return ((SColumnDefNode*)pTag)->colName; + } + return ((SColumnNode*)pTag)->colName; +} + static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SSelectStmt* pSelect) { if (NULL == pStmt->pTags) { return TSDB_CODE_SUCCESS; @@ -5612,7 +5741,7 @@ static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStream bool found = false; SNode* pPart = NULL; FOREACH(pPart, pSelect->pPartitionByList) { - if (0 == strcmp(((SColumnDefNode*)pTag)->colName, ((SExprNode*)pPart)->userAlias)) { + if (0 == strcmp(getTagNameForCreateStreamTag(pTag), ((SExprNode*)pPart)->userAlias)) { if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSelect->pTags, nodesCloneNode(pPart))) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -5621,12 +5750,32 @@ static int32_t addTagsToCreateStreamQuery(STranslateContext* pCxt, SCreateStream } } if (!found) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnDefNode*)pTag)->colName); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnDefNode*)pTag)->colName); } } return TSDB_CODE_SUCCESS; } +static SNode* createNullValue() { + SValueNode* pValue = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pValue) { + return NULL; + } + pValue->isNull = true; + pValue->node.resType.type = TSDB_DATA_TYPE_NULL; + pValue->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; + return (SNode*)pValue; +} + +static int32_t addNullTagsForExistTable(STranslateContext* pCxt, STableMeta* pMeta, SSelectStmt* pSelect) { + int32_t numOfTags = getNumOfTags(pMeta); + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfTags; ++i) { + code = nodesListMakeStrictAppend(&pSelect->pTags, createNullValue()); + } + return code; +} + typedef struct SRewriteSubtableCxt { STranslateContext* pCxt; SNodeList* pPartitionList; @@ -5672,16 +5821,30 @@ static int32_t addSubtableNameToCreateStreamQuery(STranslateContext* pCxt, SCrea return pCxt->errCode; } -static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { +static int32_t addNullTagsForCreateTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < LIST_LENGTH(pStmt->pTags); ++i) { + code = nodesListMakeStrictAppend(&((SSelectStmt*)pStmt->pQuery)->pTags, createNullValue()); + } + return code; +} + +static int32_t addNullTagsToCreateStreamQuery(STranslateContext* pCxt, STableMeta* pMeta, SCreateStreamStmt* pStmt) { + if (NULL == pMeta) { + return addNullTagsForCreateTable(pCxt, pStmt); + } + return addNullTagsForExistTable(pCxt, pMeta, (SSelectStmt*)pStmt->pQuery); +} + +static int32_t addSubtableInfoToCreateStreamQuery(STranslateContext* pCxt, STableMeta* pMeta, + SCreateStreamStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (NULL == pSelect->pPartitionByList) { - if (NULL != pStmt->pTags || NULL != pStmt->pSubtable) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); - } - return TSDB_CODE_SUCCESS; + code = addNullTagsToCreateStreamQuery(pCxt, pMeta, pStmt); + } else { + code = addTagsToCreateStreamQuery(pCxt, pStmt, pSelect); } - - int32_t code = addTagsToCreateStreamQuery(pCxt, pStmt, pSelect); if (TSDB_CODE_SUCCESS == code) { code = addSubtableNameToCreateStreamQuery(pCxt, pStmt, pSelect); } @@ -5707,9 +5870,349 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm return TSDB_CODE_SUCCESS; } +static int32_t adjustDataTypeOfProjections(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pProjections) { + if (getNumOfColumns(pMeta) != LIST_LENGTH(pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SSchema* pSchemas = getTableColumnSchema(pMeta); + int32_t index = 0; + SNode* pProj = NULL; + FOREACH(pProj, pProjections) { + SSchema* pSchema = pSchemas + index++; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pProj)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pProj, dt, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_NODE(pFunc); + } + } + + return TSDB_CODE_SUCCESS; +} + +typedef struct SProjColPos { + int32_t colId; + SNode* pProj; +} SProjColPos; + +static int32_t projColPosCompar(const void* l, const void* r) { + return ((SProjColPos*)l)->colId > ((SProjColPos*)r)->colId; +} + +static void projColPosDelete(void* p) { nodesDestroyNode(((SProjColPos*)p)->pProj); } + +static int32_t addProjToProjColPos(STranslateContext* pCxt, const SSchema* pSchema, SNode* pProj, SArray* pProjColPos) { + SNode* pNewProj = nodesCloneNode(pProj); + if (NULL == pNewProj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pNewProj)->resType)) { + SNode* pFunc = NULL; + code = createCastFunc(pCxt, pNewProj, dt, &pFunc); + pNewProj = pFunc; + } + if (TSDB_CODE_SUCCESS == code) { + SProjColPos pos = {.colId = pSchema->colId, .pProj = pNewProj}; + code = (NULL == taosArrayPush(pProjColPos, &pos) ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNewProj); + } + return code; +} + +static int32_t setFillNullCols(SArray* pProjColPos, const STableMeta* pMeta, SCMCreateStreamReq* pReq) { + int32_t numOfBoundCols = taosArrayGetSize(pProjColPos); + pReq->fillNullCols = taosArrayInit(pMeta->tableInfo.numOfColumns - numOfBoundCols, sizeof(SColLocation)); + if (NULL == pReq->fillNullCols) { + return TSDB_CODE_OUT_OF_MEMORY; + } + const SSchema* pSchemas = getTableColumnSchema(pMeta); + int32_t indexOfBoundCols = 0; + for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { + const SSchema* pSchema = pSchemas + i; + if (indexOfBoundCols < numOfBoundCols) { + SProjColPos* pPos = taosArrayGet(pProjColPos, indexOfBoundCols); + if (pSchema->colId == pPos->colId) { + ++indexOfBoundCols; + continue; + } + } + SColLocation colLoc = {.colId = pSchema->colId, .slotId = i, .type = pSchema->type}; + taosArrayPush(pReq->fillNullCols, &colLoc); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustOrderOfProjections(STranslateContext* pCxt, SNodeList* pCols, const STableMeta* pMeta, + SNodeList** pProjections, SCMCreateStreamReq* pReq) { + if (LIST_LENGTH(pCols) != LIST_LENGTH(*pProjections)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of columns"); + } + + SArray* pProjColPos = taosArrayInit(LIST_LENGTH(pCols), sizeof(SProjColPos)); + if (NULL == pProjColPos) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + bool hasPrimaryKey = false; + SNode* pCol = NULL; + SNode* pProj = NULL; + FORBOTH(pCol, pCols, pProj, *pProjections) { + const SSchema* pSchema = getNormalColSchema(pMeta, ((SColumnNode*)pCol)->colName); + if (NULL == pSchema) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, ((SColumnNode*)pCol)->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = addProjToProjColPos(pCxt, pSchema, pProj, pProjColPos); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + hasPrimaryKey = true; + } + } + + if (TSDB_CODE_SUCCESS == code && !hasPrimaryKey) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, + "primary timestamp column can not be null"); + } + + SNodeList* pNewProjections = NULL; + if (TSDB_CODE_SUCCESS == code) { + taosArraySort(pProjColPos, projColPosCompar); + int32_t num = taosArrayGetSize(pProjColPos); + pNewProjections = nodesMakeList(); + if (NULL == pNewProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + SProjColPos* pPos = taosArrayGet(pProjColPos, i); + code = nodesListStrictAppend(pNewProjections, pPos->pProj); + pPos->pProj = NULL; + } + } + + if (TSDB_CODE_SUCCESS == code && pMeta->tableInfo.numOfColumns > LIST_LENGTH(pCols)) { + code = setFillNullCols(pProjColPos, pMeta, pReq); + } + + if (TSDB_CODE_SUCCESS == code) { + taosArrayDestroy(pProjColPos); + nodesDestroyList(*pProjections); + *pProjections = pNewProjections; + } else { + taosArrayDestroyEx(pProjColPos, projColPosDelete); + nodesDestroyList(pNewProjections); + } + + return code; +} + +static int32_t adjustProjectionsForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, + const STableMeta* pMeta, SCMCreateStreamReq* pReq) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pStmt->pCols) { + return adjustDataTypeOfProjections(pCxt, pMeta, pSelect->pProjectionList); + } + return adjustOrderOfProjections(pCxt, pStmt->pCols, pMeta, &pSelect->pProjectionList, pReq); +} + +static bool isGroupIdTagStream(const STableMeta* pMeta, SNodeList* pTags) { + return (NULL == pTags && 1 == pMeta->tableInfo.numOfTags && TSDB_DATA_TYPE_UBIGINT == getTableTagSchema(pMeta)->type); +} + +static int32_t adjustDataTypeOfTags(STranslateContext* pCxt, const STableMeta* pMeta, SNodeList* pTags) { + if (isGroupIdTagStream(pMeta, pTags)) { + return TSDB_CODE_SUCCESS; + } + + if (getNumOfTags(pMeta) != LIST_LENGTH(pTags)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of tags"); + } + + SSchema* pSchemas = getTableTagSchema(pMeta); + int32_t index = 0; + SNode* pTag = NULL; + FOREACH(pTag, pTags) { + SSchema* pSchema = pSchemas + index++; + SDataType dt = {.type = pSchema->type, .bytes = pSchema->bytes}; + if (!dataTypeEqual(&dt, &((SExprNode*)pTag)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pTag, dt, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_NODE(pFunc); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustOrderOfTags(STranslateContext* pCxt, SNodeList* pTags, const STableMeta* pMeta, + SNodeList** pTagExprs, SCMCreateStreamReq* pReq) { + if (LIST_LENGTH(pTags) != LIST_LENGTH(*pTagExprs)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM, "Illegal number of tags"); + } + + SArray* pTagPos = taosArrayInit(LIST_LENGTH(pTags), sizeof(SProjColPos)); + if (NULL == pTagPos) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + SNode* pTag = NULL; + SNode* pTagExpr = NULL; + FORBOTH(pTag, pTags, pTagExpr, *pTagExprs) { + const SSchema* pSchema = getTagSchema(pMeta, ((SColumnNode*)pTag)->colName); + if (NULL == pSchema) { + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, ((SColumnNode*)pTag)->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = addProjToProjColPos(pCxt, pSchema, pTagExpr, pTagPos); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + SNodeList* pNewTagExprs = NULL; + if (TSDB_CODE_SUCCESS == code) { + taosArraySort(pTagPos, projColPosCompar); + int32_t indexOfBoundTags = 0; + int32_t numOfBoundTags = taosArrayGetSize(pTagPos); + int32_t numOfTags = getNumOfTags(pMeta); + const SSchema* pTagsSchema = getTableTagSchema(pMeta); + pNewTagExprs = nodesMakeList(); + if (NULL == pNewTagExprs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; TSDB_CODE_SUCCESS == code && i < numOfTags; ++i) { + const SSchema* pTagSchema = pTagsSchema + i; + if (indexOfBoundTags < numOfBoundTags) { + SProjColPos* pPos = taosArrayGet(pTagPos, indexOfBoundTags); + if (pPos->colId == pTagSchema->colId) { + ++indexOfBoundTags; + code = nodesListStrictAppend(pNewTagExprs, pPos->pProj); + pPos->pProj = NULL; + continue; + } + } + code = nodesListStrictAppend(pNewTagExprs, createNullValue()); + } + } + + if (TSDB_CODE_SUCCESS == code) { + taosArrayDestroy(pTagPos); + nodesDestroyList(*pTagExprs); + *pTagExprs = pNewTagExprs; + } else { + taosArrayDestroyEx(pTagPos, projColPosDelete); + nodesDestroyList(pNewTagExprs); + } + + return code; +} + +static int32_t adjustTagsForExistTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, const STableMeta* pMeta, + SCMCreateStreamReq* pReq) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pSelect->pPartitionByList) { + return TSDB_CODE_SUCCESS; + } + if (NULL == pStmt->pTags) { + return adjustDataTypeOfTags(pCxt, pMeta, pSelect->pTags); + } + return adjustOrderOfTags(pCxt, pStmt->pTags, pMeta, &pSelect->pTags, pReq); +} + +static int32_t adjustTagsForCreateTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (NULL == pSelect->pPartitionByList || NULL == pSelect->pTags) { + return TSDB_CODE_SUCCESS; + } + + SNode* pTagDef = NULL; + SNode* pTagExpr = NULL; + FORBOTH(pTagDef, pStmt->pTags, pTagExpr, pSelect->pTags) { + SColumnDefNode* pDef = (SColumnDefNode*)pTagDef; + if (!dataTypeEqual(&pDef->dataType, &((SExprNode*)pTagExpr)->resType)) { + SNode* pFunc = NULL; + int32_t code = createCastFunc(pCxt, pTagExpr, pDef->dataType, &pFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST2_NODE(pFunc); + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t adjustTags(STranslateContext* pCxt, SCreateStreamStmt* pStmt, const STableMeta* pMeta, + SCMCreateStreamReq* pReq) { + if (NULL == pMeta) { + return adjustTagsForCreateTable(pCxt, pStmt, pReq); + } + return adjustTagsForExistTable(pCxt, pStmt, pMeta, pReq); +} + +static bool isTagDef(SNodeList* pTags) { + if (NULL == pTags) { + return false; + } + return QUERY_NODE_COLUMN_DEF == nodeType(nodesListGetNode(pTags, 0)); +} + +static bool isTagBound(SNodeList* pTags) { + if (NULL == pTags) { + return false; + } + return QUERY_NODE_COLUMN == nodeType(nodesListGetNode(pTags, 0)); +} + +static int32_t translateStreamTargetTable(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq, + STableMeta** pMeta) { + int32_t code = getTableMeta(pCxt, pStmt->targetDbName, pStmt->targetTabName, pMeta); + if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { + if (NULL != pStmt->pCols || isTagBound(pStmt->pTags)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pStmt->targetTabName); + } + pReq->createStb = STREAM_CREATE_STABLE_TRUE; + pReq->targetStbUid = 0; + return TSDB_CODE_SUCCESS; + } else if (TSDB_CODE_SUCCESS == code) { + if (isTagDef(pStmt->pTags)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Table already exist: %s", + pStmt->targetTabName); + } + if (TSDB_SUPER_TABLE != (*pMeta)->tableType) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, + "Stream can only be written to super table"); + } + pReq->createStb = STREAM_CREATE_STABLE_FALSE; + pReq->targetStbUid = (*pMeta)->suid; + } + return code; +} + static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStmt, SCMCreateStreamReq* pReq) { pCxt->createStream = true; - int32_t code = addSubtableInfoToCreateStreamQuery(pCxt, pStmt); + STableMeta* pMeta = NULL; + int32_t code = translateStreamTargetTable(pCxt, pStmt, pReq, &pMeta); + if (TSDB_CODE_SUCCESS == code) { + code = addSubtableInfoToCreateStreamQuery(pCxt, pMeta, pStmt); + } if (TSDB_CODE_SUCCESS == code) { code = translateQuery(pCxt, pStmt->pQuery); } @@ -5719,10 +6222,17 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt if (TSDB_CODE_SUCCESS == code) { code = checkStreamQuery(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code && NULL != pMeta) { + code = adjustProjectionsForExistTable(pCxt, pStmt, pMeta, pReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = adjustTags(pCxt, pStmt, pMeta, pReq); + } if (TSDB_CODE_SUCCESS == code) { getSourceDatabase(pStmt->pQuery, pCxt->pParseCxt->acctId, pReq->sourceDB); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } + taosMemoryFree(pMeta); return code; } @@ -5752,7 +6262,7 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* int32_t code = buildCreateStreamQuery(pCxt, pStmt, pReq); if (TSDB_CODE_SUCCESS == code) { - pReq->sql = strdup(pCxt->pParseCxt->pSql); + pReq->sql = taosStrdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { code = TSDB_CODE_OUT_OF_MEMORY; } @@ -5770,9 +6280,11 @@ static int32_t buildCreateStreamReq(STranslateContext* pCxt, SCreateStreamStmt* (NULL != pStmt->pOptions->pDeleteMark ? ((SValueNode*)pStmt->pOptions->pDeleteMark)->datum.i : 0); pReq->fillHistory = pStmt->pOptions->fillHistory; pReq->igExpired = pStmt->pOptions->ignoreExpired; + if (pReq->createStb) { + columnDefNodeToField(pStmt->pTags, &pReq->pTags); + pReq->numOfTags = LIST_LENGTH(pStmt->pTags); + } pReq->igUpdate = pStmt->pOptions->ignoreUpdate; - columnDefNodeToField(pStmt->pTags, &pReq->pTags); - pReq->numOfTags = LIST_LENGTH(pStmt->pTags); } return code; @@ -6076,6 +6588,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_DESCRIBE_STMT: code = translateDescribe(pCxt, (SDescribeStmt*)pNode); break; + case QUERY_NODE_COMPACT_DATABASE_STMT: + code = translateCompact(pCxt, (SCompactDatabaseStmt*)pNode); + break; case QUERY_NODE_KILL_CONNECTION_STMT: code = translateKillConnection(pCxt, (SKillStmt*)pNode); break; @@ -6231,6 +6746,20 @@ static int32_t extractShowCreateDatabaseResultSchema(int32_t* numOfCols, SSchema return TSDB_CODE_SUCCESS; } +static int32_t extractShowAliveResultSchema(int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = 1; + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pSchema)[0].type = TSDB_DATA_TYPE_INT; + (*pSchema)[0].bytes = sizeof(int32_t); + strcpy((*pSchema)[0].name, "status"); + + return TSDB_CODE_SUCCESS; +} + static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** pSchema) { *numOfCols = 2; *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); @@ -6282,6 +6811,9 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS return extractDescribeResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return extractShowCreateDatabaseResultSchema(numOfCols, pSchema); + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + return extractShowAliveResultSchema(numOfCols, pSchema); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return extractShowCreateTableResultSchema(numOfCols, pSchema); @@ -6386,7 +6918,7 @@ static int32_t createOperatorNode(EOperatorType opType, const char* pColName, SN nodesDestroyNode((SNode*)pOper); return TSDB_CODE_OUT_OF_MEMORY; } - strcpy(((SColumnNode*)pOper->pLeft)->colName, pColName); + snprintf(((SColumnNode*)pOper->pLeft)->colName, sizeof(((SColumnNode*)pOper->pLeft)->colName), "%s", pColName); *pOp = (SNode*)pOper; return TSDB_CODE_SUCCESS; @@ -6622,10 +7154,10 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* SVCreateTbReq req = {0}; req.type = TD_NORMAL_TABLE; - req.name = strdup(pStmt->tableName); + req.name = taosStrdup(pStmt->tableName); req.ttl = pStmt->pOptions->ttl; if (pStmt->pOptions->commentNull == false) { - req.comment = strdup(pStmt->pOptions->comment); + req.comment = taosStrdup(pStmt->pOptions->comment); if (NULL == req.comment) { tdDestroySVCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; @@ -6706,7 +7238,7 @@ static void destroyCreateTbReqBatch(void* data) { } int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray) { - SVnodeModifOpStmt* pNewStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); + SVnodeModifyOpStmt* pNewStmt = (SVnodeModifyOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT); if (pNewStmt == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -6784,17 +7316,17 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S struct SVCreateTbReq req = {0}; req.type = TD_CHILD_TABLE; - req.name = strdup(pStmt->tableName); + req.name = taosStrdup(pStmt->tableName); req.ttl = pStmt->pOptions->ttl; if (pStmt->pOptions->commentNull == false) { - req.comment = strdup(pStmt->pOptions->comment); + req.comment = taosStrdup(pStmt->pOptions->comment); req.commentLen = strlen(pStmt->pOptions->comment); } else { req.commentLen = -1; } req.ctb.suid = suid; req.ctb.tagNum = tagNum; - req.ctb.stbName = strdup(sTableNmae); + req.ctb.stbName = taosStrdup(sTableNmae); req.ctb.pTag = (uint8_t*)pTag; req.ctb.tagName = taosArrayDup(tagName, NULL); if (pStmt->ignoreExists) { @@ -7090,7 +7622,7 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap) { } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; + SCreateMultiTablesStmt* pStmt = (SCreateMultiTablesStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == pVgroupHashmap) { @@ -7276,7 +7808,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pStmt->colName); } - pReq->tagName = strdup(pStmt->colName); + pReq->tagName = taosStrdup(pStmt->colName); if (NULL == pReq->tagName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7349,7 +7881,7 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); } - pReq->colName = strdup(pStmt->colName); + pReq->colName = taosStrdup(pStmt->colName); if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7360,19 +7892,19 @@ static int32_t buildAddColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, S return TSDB_CODE_SUCCESS; } -static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { if (2 == getNumOfColumns(pTableMeta)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DROP_COL); } - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_CANNOT_DROP_PRIMARY_KEY); } - pReq->colName = strdup(pStmt->colName); + pReq->colName = taosStrdup(pStmt->colName); if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7381,11 +7913,11 @@ static int32_t buildDropColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, return TSDB_CODE_SUCCESS; } -static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, +static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, const STableMeta* pTableMeta, SVAlterTbReq* pReq) { pReq->colModBytes = calcTypeBytes(pStmt->dataType); pReq->colModType = pStmt->dataType.type; - SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); + const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || @@ -7397,7 +7929,7 @@ static int32_t buildUpdateColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROW_LENGTH, TSDB_MAX_BYTES_PER_ROW); } - pReq->colName = strdup(pStmt->colName); + pReq->colName = taosStrdup(pStmt->colName); if (NULL == pReq->colName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7415,8 +7947,8 @@ static int32_t buildRenameColReq(STranslateContext* pCxt, SAlterTableStmt* pStmt return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_DUPLICATED_COLUMN); } - pReq->colName = strdup(pStmt->colName); - pReq->colNewName = strdup(pStmt->newColName); + pReq->colName = taosStrdup(pStmt->colName); + pReq->colNewName = taosStrdup(pStmt->newColName); if (NULL == pReq->colName || NULL == pReq->colNewName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7433,7 +7965,7 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p if (TSDB_CODE_SUCCESS == code) { if (pStmt->pOptions->commentNull == false) { - pReq->newComment = strdup(pStmt->pOptions->comment); + pReq->newComment = taosStrdup(pStmt->pOptions->comment); if (NULL == pReq->newComment) { code = TSDB_CODE_OUT_OF_MEMORY; } else { @@ -7449,7 +7981,7 @@ static int32_t buildUpdateOptionsReq(STranslateContext* pCxt, SAlterTableStmt* p static int32_t buildAlterTbReq(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SVAlterTbReq* pReq) { - pReq->tbName = strdup(pStmt->tableName); + pReq->tbName = taosStrdup(pStmt->tableName); if (NULL == pReq->tbName) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -7698,7 +8230,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { code = rewriteCreateTable(pCxt, pQuery); } break; - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: code = rewriteCreateMultiTable(pCxt, pQuery); break; case QUERY_NODE_DROP_TABLE_STMT: @@ -7795,12 +8327,14 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_SUBMIT; break; - case QUERY_NODE_VNODE_MODIF_STMT: + case QUERY_NODE_VNODE_MODIFY_STMT: pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = toMsgType(((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType); + pQuery->msgType = toMsgType(((SVnodeModifyOpStmt*)pQuery->pRoot)->sqlNodeType); break; case QUERY_NODE_DESCRIBE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: + case QUERY_NODE_SHOW_DB_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index fa091901b680041721936f7501fe23bb53b36305..89a42cb03c29fcc97570610d52b5bc15a663fc9b 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -971,7 +971,7 @@ static SArray* smaIndexesDup(SArray* pSrc) { } for (int32_t i = 0; i < size; ++i) { STableIndexInfo* pIndex = taosArrayGet(pDst, i); - pIndex->expr = taosMemoryStrDup(((STableIndexInfo*)taosArrayGet(pSrc, i))->expr); + pIndex->expr = taosStrdup(((STableIndexInfo*)taosArrayGet(pSrc, i))->expr); if (NULL == pIndex->expr) { taosArrayDestroyEx(pDst, destroySmaIndex); return NULL; diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 98368d38063671eb741b2183dd1daea4c601aa4a..b6940f83959260b3caa20cea696b8f83ca74231b 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 461 +#define YYNOCODE 469 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy32; - SToken yy77; - int32_t yy248; - int8_t yy287; - ENullOrder yy385; - EJoinType yy560; - SNode* yy600; - SNodeList* yy601; - SAlterOption yy661; - EOperatorType yy666; - int64_t yy717; - EFillMode yy798; - bool yy841; - SDataType yy888; + EOperatorType yy2; + SNode* yy42; + bool yy103; + EOrder yy106; + SNodeList* yy110; + SToken yy225; + EFillMode yy410; + SDataType yy448; + SAlterOption yy459; + int32_t yy508; + ENullOrder yy599; + EJoinType yy638; + int64_t yy641; + int8_t yy705; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 716 -#define YYNRULE 545 -#define YYNTOKEN 324 -#define YY_MAX_SHIFT 715 -#define YY_MIN_SHIFTREDUCE 1063 -#define YY_MAX_SHIFTREDUCE 1607 -#define YY_ERROR_ACTION 1608 -#define YY_ACCEPT_ACTION 1609 -#define YY_NO_ACTION 1610 -#define YY_MIN_REDUCE 1611 -#define YY_MAX_REDUCE 2155 +#define YYNSTATE 743 +#define YYNRULE 563 +#define YYNTOKEN 328 +#define YY_MAX_SHIFT 742 +#define YY_MIN_SHIFTREDUCE 1102 +#define YY_MAX_SHIFTREDUCE 1664 +#define YY_ERROR_ACTION 1665 +#define YY_ACCEPT_ACTION 1666 +#define YY_NO_ACTION 1667 +#define YY_MIN_REDUCE 1668 +#define YY_MAX_REDUCE 2230 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,744 +216,753 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2733) +#define YY_ACTTAB_COUNT (2749) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 463, 370, 464, 1647, 1810, 1812, 404, 1969, 1755, 33, - /* 10 */ 277, 170, 43, 41, 1536, 1634, 1753, 571, 168, 1951, - /* 20 */ 365, 2126, 1386, 1955, 1804, 42, 40, 39, 38, 37, - /* 30 */ 1969, 605, 1609, 1466, 1951, 1384, 570, 174, 1987, 335, - /* 40 */ 1864, 2127, 572, 1811, 1812, 52, 622, 591, 550, 1947, - /* 50 */ 1953, 1937, 2126, 621, 11, 10, 1411, 589, 1461, 1937, - /* 60 */ 615, 1987, 1764, 16, 1947, 1953, 347, 2132, 174, 586, - /* 70 */ 1392, 1412, 2127, 572, 1937, 615, 621, 462, 1968, 604, - /* 80 */ 467, 1653, 2003, 43, 41, 161, 1970, 625, 1972, 1973, - /* 90 */ 620, 365, 615, 1386, 479, 12, 1873, 380, 604, 550, - /* 100 */ 605, 1968, 159, 2126, 1466, 2003, 1384, 1718, 101, 1970, - /* 110 */ 625, 1972, 1973, 620, 124, 615, 260, 712, 2132, 174, - /* 120 */ 171, 502, 2056, 2127, 572, 229, 359, 2052, 1794, 1461, - /* 130 */ 604, 1764, 1468, 1469, 16, 97, 605, 573, 2147, 550, - /* 140 */ 176, 1392, 583, 2126, 472, 94, 464, 1647, 2082, 1509, - /* 150 */ 124, 157, 1182, 59, 46, 86, 297, 507, 2132, 174, - /* 160 */ 592, 1442, 1451, 2127, 572, 471, 12, 1764, 467, 1653, - /* 170 */ 295, 67, 356, 132, 66, 1877, 2071, 517, 516, 515, - /* 180 */ 1387, 1680, 1385, 1265, 1266, 129, 511, 1184, 712, 1956, - /* 190 */ 510, 357, 194, 459, 457, 509, 514, 1318, 1319, 156, - /* 200 */ 1951, 508, 2068, 1468, 1469, 46, 1390, 1391, 1766, 1441, - /* 210 */ 1444, 1445, 1446, 1447, 1448, 1449, 1450, 617, 613, 1459, - /* 220 */ 1460, 1462, 1463, 1464, 1465, 1467, 1470, 2, 59, 605, - /* 230 */ 1947, 1953, 1442, 1451, 257, 2064, 582, 590, 125, 581, - /* 240 */ 1413, 615, 2126, 179, 469, 1411, 517, 516, 515, 168, - /* 250 */ 465, 1387, 227, 1385, 129, 511, 226, 570, 174, 510, - /* 260 */ 1764, 50, 2127, 572, 509, 514, 549, 100, 36, 35, - /* 270 */ 508, 1865, 42, 40, 39, 38, 37, 1390, 1391, 378, - /* 280 */ 1441, 1444, 1445, 1446, 1447, 1448, 1449, 1450, 617, 613, - /* 290 */ 1459, 1460, 1462, 1463, 1464, 1465, 1467, 1470, 2, 522, - /* 300 */ 9, 43, 41, 177, 177, 75, 74, 407, 92, 365, - /* 310 */ 181, 1386, 83, 323, 532, 1987, 345, 98, 533, 1969, - /* 320 */ 59, 550, 1466, 565, 1384, 2126, 1817, 145, 225, 319, - /* 330 */ 1757, 133, 395, 358, 393, 389, 385, 382, 379, 1756, - /* 340 */ 2132, 174, 1815, 525, 1955, 2127, 572, 1461, 519, 368, - /* 350 */ 1987, 63, 16, 224, 371, 1951, 1574, 156, 622, 1392, - /* 360 */ 2131, 481, 156, 1937, 564, 621, 1766, 39, 38, 37, - /* 370 */ 591, 1766, 43, 41, 1471, 566, 59, 1413, 177, 51, - /* 380 */ 365, 1969, 1386, 259, 12, 1947, 1953, 348, 59, 65, - /* 390 */ 1968, 81, 64, 1466, 2003, 1384, 615, 101, 1970, 625, - /* 400 */ 1972, 1973, 620, 9, 615, 128, 712, 135, 1497, 142, - /* 410 */ 2027, 2056, 1987, 1742, 1759, 359, 2052, 600, 1461, 1873, - /* 420 */ 622, 1468, 1469, 268, 269, 1937, 1096, 621, 267, 1740, - /* 430 */ 1392, 1222, 647, 646, 645, 1226, 644, 1228, 1229, 643, - /* 440 */ 1231, 640, 59, 1237, 637, 1239, 1240, 634, 631, 650, - /* 450 */ 1442, 1451, 1968, 1597, 1412, 44, 2003, 1611, 186, 102, - /* 460 */ 1970, 625, 1972, 1973, 620, 1098, 615, 1101, 1102, 1387, - /* 470 */ 177, 1385, 403, 2056, 402, 481, 1411, 712, 2053, 410, - /* 480 */ 30, 123, 122, 121, 120, 119, 118, 117, 116, 115, - /* 490 */ 1502, 659, 1468, 1469, 78, 1390, 1391, 77, 1441, 1444, - /* 500 */ 1445, 1446, 1447, 1448, 1449, 1450, 617, 613, 1459, 1460, - /* 510 */ 1462, 1463, 1464, 1465, 1467, 1470, 2, 513, 512, 321, - /* 520 */ 259, 1442, 1451, 1612, 114, 605, 177, 113, 112, 111, - /* 530 */ 110, 109, 108, 107, 106, 105, 81, 659, 177, 408, - /* 540 */ 1387, 158, 1385, 1623, 114, 1410, 1633, 113, 112, 111, - /* 550 */ 110, 109, 108, 107, 106, 105, 1764, 36, 35, 1760, - /* 560 */ 1969, 42, 40, 39, 38, 37, 1390, 1391, 47, 1441, - /* 570 */ 1444, 1445, 1446, 1447, 1448, 1449, 1450, 617, 613, 1459, - /* 580 */ 1460, 1462, 1463, 1464, 1465, 1467, 1470, 2, 43, 41, - /* 590 */ 1937, 1987, 177, 25, 411, 446, 365, 592, 1386, 586, - /* 600 */ 340, 1115, 1392, 1114, 1937, 561, 621, 412, 9, 1466, - /* 610 */ 7, 1384, 1878, 1817, 1817, 607, 211, 2028, 36, 35, - /* 620 */ 369, 332, 42, 40, 39, 38, 37, 6, 1632, 1815, - /* 630 */ 1815, 1968, 1116, 1955, 1461, 2003, 531, 583, 101, 1970, - /* 640 */ 625, 1972, 1973, 620, 1951, 615, 1392, 1563, 605, 529, - /* 650 */ 171, 527, 2056, 190, 189, 583, 359, 2052, 2071, 43, - /* 660 */ 41, 341, 409, 339, 338, 671, 504, 365, 132, 1386, - /* 670 */ 506, 44, 1937, 328, 1947, 1953, 360, 1749, 2083, 1764, - /* 680 */ 1466, 399, 1384, 1414, 2067, 615, 132, 571, 1540, 567, - /* 690 */ 562, 2126, 505, 712, 1411, 558, 557, 1561, 1562, 1564, - /* 700 */ 1565, 1566, 545, 401, 397, 1461, 570, 174, 1468, 1469, - /* 710 */ 1631, 2127, 572, 228, 1495, 2131, 1630, 1392, 177, 2126, - /* 720 */ 36, 35, 605, 1751, 42, 40, 39, 38, 37, 173, - /* 730 */ 2064, 2065, 1414, 130, 2069, 2130, 418, 1442, 1451, 2127, - /* 740 */ 2129, 2131, 12, 2071, 1476, 2126, 585, 172, 2064, 2065, - /* 750 */ 1411, 130, 2069, 1764, 1937, 1747, 1387, 609, 1385, 2028, - /* 760 */ 1937, 2130, 506, 1629, 712, 2127, 2128, 36, 35, 2066, - /* 770 */ 1496, 42, 40, 39, 38, 37, 1361, 1362, 134, 1468, - /* 780 */ 1469, 2027, 1390, 1391, 505, 1441, 1444, 1445, 1446, 1447, - /* 790 */ 1448, 1449, 1450, 617, 613, 1459, 1460, 1462, 1463, 1464, - /* 800 */ 1465, 1467, 1470, 2, 320, 2130, 1409, 1937, 1442, 1451, - /* 810 */ 1115, 232, 1114, 440, 683, 681, 453, 36, 35, 452, - /* 820 */ 1860, 42, 40, 39, 38, 37, 183, 1387, 1677, 1385, - /* 830 */ 651, 182, 1529, 1808, 424, 1443, 454, 1628, 1533, 426, - /* 840 */ 1627, 1116, 32, 363, 1490, 1491, 1492, 1493, 1494, 1498, - /* 850 */ 1499, 1500, 1501, 1390, 1391, 1552, 1441, 1444, 1445, 1446, - /* 860 */ 1447, 1448, 1449, 1450, 617, 613, 1459, 1460, 1462, 1463, - /* 870 */ 1464, 1465, 1467, 1470, 2, 237, 29, 1626, 616, 1625, - /* 880 */ 336, 1937, 36, 35, 1937, 649, 42, 40, 39, 38, - /* 890 */ 37, 1443, 414, 689, 688, 687, 686, 375, 1622, 685, - /* 900 */ 684, 136, 679, 678, 677, 676, 675, 674, 673, 149, - /* 910 */ 669, 668, 667, 374, 373, 664, 663, 662, 661, 660, - /* 920 */ 1621, 1937, 450, 1937, 1969, 445, 444, 443, 442, 439, - /* 930 */ 438, 437, 436, 435, 431, 430, 429, 428, 337, 421, - /* 940 */ 420, 419, 1937, 416, 415, 334, 1620, 1969, 1619, 1860, - /* 950 */ 1860, 291, 362, 361, 1794, 1987, 1618, 605, 605, 1414, - /* 960 */ 184, 188, 1400, 622, 1937, 605, 605, 1617, 1937, 1741, - /* 970 */ 621, 432, 433, 1466, 156, 1393, 1411, 583, 1987, 372, - /* 980 */ 480, 1604, 672, 1767, 1734, 1817, 622, 1624, 1764, 1764, - /* 990 */ 1937, 1937, 1937, 621, 377, 1968, 1764, 1764, 1461, 2003, - /* 1000 */ 1937, 1816, 101, 1970, 625, 1972, 1973, 620, 132, 615, - /* 1010 */ 1392, 1937, 1101, 1102, 2146, 1616, 2056, 1719, 1968, 1615, - /* 1020 */ 359, 2052, 2003, 1847, 1614, 101, 1970, 625, 1972, 1973, - /* 1030 */ 620, 2090, 615, 1969, 575, 31, 550, 2146, 2096, 2056, - /* 1040 */ 2126, 36, 35, 359, 2052, 42, 40, 39, 38, 37, - /* 1050 */ 655, 2076, 1529, 1808, 2120, 2132, 174, 611, 1924, 1937, - /* 1060 */ 2127, 572, 656, 1937, 1987, 1808, 574, 212, 1937, 175, - /* 1070 */ 2064, 2065, 622, 130, 2069, 210, 657, 1937, 1603, 621, - /* 1080 */ 1532, 1667, 163, 1660, 48, 138, 3, 126, 498, 494, - /* 1090 */ 490, 486, 209, 68, 254, 147, 146, 654, 653, 652, - /* 1100 */ 144, 578, 236, 518, 1968, 520, 387, 535, 2003, 534, - /* 1110 */ 1969, 101, 1970, 625, 1972, 1973, 620, 1443, 615, 605, - /* 1120 */ 1401, 217, 1396, 2146, 215, 2056, 1606, 1607, 82, 359, - /* 1130 */ 2052, 207, 1395, 1761, 1969, 665, 219, 235, 1386, 218, - /* 1140 */ 2075, 1987, 61, 76, 61, 241, 1404, 1406, 657, 622, - /* 1150 */ 1764, 1384, 605, 1658, 1937, 1394, 621, 1163, 613, 1459, - /* 1160 */ 1460, 1462, 1463, 1464, 1465, 1987, 140, 147, 146, 654, - /* 1170 */ 653, 652, 144, 622, 427, 523, 84, 612, 1937, 1969, - /* 1180 */ 621, 1968, 559, 1764, 221, 2003, 1392, 220, 101, 1970, - /* 1190 */ 625, 1972, 1973, 620, 1560, 615, 1559, 243, 206, 200, - /* 1200 */ 2031, 205, 2056, 45, 477, 1968, 359, 2052, 248, 2003, - /* 1210 */ 1987, 1739, 101, 1970, 625, 1972, 1973, 620, 622, 615, - /* 1220 */ 198, 11, 10, 1937, 2029, 621, 2056, 1988, 376, 265, - /* 1230 */ 359, 2052, 605, 712, 1969, 36, 35, 605, 1869, 42, - /* 1240 */ 40, 39, 38, 37, 1648, 141, 546, 143, 145, 61, - /* 1250 */ 1968, 587, 605, 605, 2003, 1331, 576, 101, 1970, 625, - /* 1260 */ 1972, 1973, 620, 1764, 615, 1987, 272, 602, 1764, 608, - /* 1270 */ 1805, 2056, 605, 622, 223, 359, 2052, 222, 1937, 1398, - /* 1280 */ 621, 270, 538, 1764, 1764, 2086, 603, 36, 35, 584, - /* 1290 */ 1969, 42, 40, 39, 38, 37, 1387, 597, 1385, 274, - /* 1300 */ 1215, 1503, 1397, 1764, 45, 1968, 1958, 45, 629, 2003, - /* 1310 */ 143, 145, 102, 1970, 625, 1972, 1973, 620, 657, 615, - /* 1320 */ 1487, 1987, 1390, 1391, 550, 579, 2056, 1654, 2126, 622, - /* 1330 */ 2055, 2052, 127, 605, 1937, 1969, 621, 147, 146, 654, - /* 1340 */ 653, 652, 144, 2132, 174, 666, 143, 278, 2127, 572, - /* 1350 */ 1144, 256, 253, 4, 1960, 1, 1452, 381, 1969, 290, - /* 1360 */ 1243, 1968, 1247, 1254, 1764, 2003, 1987, 1161, 102, 1970, - /* 1370 */ 625, 1972, 1973, 620, 622, 615, 707, 386, 333, 1937, - /* 1380 */ 1348, 621, 2056, 285, 1252, 1145, 610, 2052, 187, 1987, - /* 1390 */ 155, 413, 1414, 1870, 417, 448, 422, 619, 148, 1409, - /* 1400 */ 434, 1862, 1937, 441, 621, 447, 623, 455, 449, 191, - /* 1410 */ 2003, 456, 458, 102, 1970, 625, 1972, 1973, 620, 460, - /* 1420 */ 615, 1415, 461, 470, 1417, 473, 474, 2056, 197, 1968, - /* 1430 */ 199, 327, 2052, 2003, 1416, 475, 313, 1970, 625, 1972, - /* 1440 */ 1973, 620, 618, 615, 606, 2021, 1969, 1418, 202, 476, - /* 1450 */ 204, 478, 79, 80, 208, 1118, 482, 501, 539, 1915, - /* 1460 */ 503, 499, 1969, 500, 537, 322, 1754, 104, 214, 540, - /* 1470 */ 286, 2087, 541, 230, 1912, 1750, 216, 1987, 544, 233, - /* 1480 */ 547, 560, 150, 595, 151, 622, 2102, 1752, 1748, 152, - /* 1490 */ 1937, 153, 621, 1987, 5, 2101, 2078, 569, 1911, 247, - /* 1500 */ 555, 622, 553, 164, 249, 552, 1937, 251, 621, 250, - /* 1510 */ 554, 350, 556, 349, 563, 2149, 580, 1968, 1969, 2097, - /* 1520 */ 577, 2003, 1529, 239, 160, 1970, 625, 1972, 1973, 620, - /* 1530 */ 242, 615, 131, 1968, 2125, 255, 1413, 2003, 261, 1419, - /* 1540 */ 160, 1970, 625, 1972, 1973, 620, 1969, 615, 252, 1987, - /* 1550 */ 2072, 588, 1874, 353, 593, 287, 288, 622, 594, 1883, - /* 1560 */ 1882, 1881, 1937, 355, 621, 551, 2093, 598, 89, 599, - /* 1570 */ 58, 91, 289, 1765, 2037, 93, 627, 1987, 1735, 1809, - /* 1580 */ 292, 281, 2094, 709, 708, 622, 49, 711, 301, 1968, - /* 1590 */ 1937, 316, 621, 2003, 294, 315, 307, 1970, 625, 1972, - /* 1600 */ 1973, 620, 324, 615, 305, 296, 1931, 1930, 1969, 325, - /* 1610 */ 72, 1929, 1928, 73, 1925, 383, 384, 1968, 1378, 1379, - /* 1620 */ 715, 2003, 180, 1923, 161, 1970, 625, 1972, 1973, 620, - /* 1630 */ 388, 615, 390, 391, 284, 392, 1922, 394, 1921, 1987, - /* 1640 */ 568, 1920, 398, 396, 354, 1919, 1351, 622, 400, 167, - /* 1650 */ 1350, 1894, 1937, 1893, 621, 705, 701, 697, 693, 282, - /* 1660 */ 405, 1892, 406, 1969, 1309, 1855, 1854, 1852, 137, 1851, - /* 1670 */ 1850, 1853, 1849, 1848, 1846, 1845, 1844, 2148, 1891, 1968, - /* 1680 */ 185, 423, 1843, 2003, 425, 1842, 314, 1970, 625, 1972, - /* 1690 */ 1973, 620, 1841, 615, 1987, 99, 1840, 1839, 275, 1838, - /* 1700 */ 1837, 1836, 619, 1835, 1834, 1833, 1832, 1937, 1831, 621, - /* 1710 */ 1830, 1829, 1828, 1827, 1826, 1825, 1824, 1823, 1822, 139, - /* 1720 */ 1311, 1821, 1820, 451, 1819, 1969, 1190, 1682, 1681, 1679, - /* 1730 */ 1643, 601, 1818, 70, 1968, 195, 1957, 192, 2003, 193, - /* 1740 */ 1642, 313, 1970, 625, 1972, 1973, 620, 1104, 615, 466, - /* 1750 */ 2022, 169, 1103, 1907, 1901, 1969, 1987, 1890, 1889, 1872, - /* 1760 */ 196, 364, 71, 203, 622, 1743, 468, 1678, 262, 1937, - /* 1770 */ 201, 621, 1676, 1137, 483, 485, 1674, 487, 1672, 491, - /* 1780 */ 489, 1670, 484, 488, 1657, 1355, 1987, 231, 493, 497, - /* 1790 */ 495, 366, 1656, 492, 622, 496, 1968, 1639, 1745, 1937, - /* 1800 */ 2003, 621, 1258, 314, 1970, 625, 1972, 1973, 620, 1259, - /* 1810 */ 615, 1744, 1181, 1969, 1180, 680, 1173, 1179, 1178, 682, - /* 1820 */ 1175, 1174, 1172, 1668, 342, 1661, 1968, 213, 343, 1969, - /* 1830 */ 2003, 1659, 344, 314, 1970, 625, 1972, 1973, 620, 521, - /* 1840 */ 615, 60, 524, 1638, 1987, 1637, 1636, 526, 528, 530, - /* 1850 */ 1366, 1368, 622, 103, 1365, 1906, 1370, 1937, 24, 621, - /* 1860 */ 1987, 1357, 1900, 154, 542, 1888, 1886, 2131, 622, 1576, - /* 1870 */ 14, 18, 17, 1937, 1969, 621, 56, 62, 57, 245, - /* 1880 */ 15, 28, 19, 26, 536, 238, 246, 240, 2003, 1558, - /* 1890 */ 1969, 309, 1970, 625, 1972, 1973, 620, 1551, 615, 162, - /* 1900 */ 1968, 1958, 1591, 244, 2003, 1987, 53, 298, 1970, 625, - /* 1910 */ 1972, 1973, 620, 622, 615, 543, 234, 548, 1937, 27, - /* 1920 */ 621, 1987, 1590, 351, 346, 1596, 258, 1595, 1594, 622, - /* 1930 */ 352, 85, 165, 54, 1937, 1887, 621, 1597, 55, 1885, - /* 1940 */ 1526, 1884, 1871, 1525, 263, 1968, 20, 88, 264, 2003, - /* 1950 */ 276, 266, 299, 1970, 625, 1972, 1973, 620, 271, 615, - /* 1960 */ 21, 1968, 596, 273, 1556, 2003, 87, 1969, 300, 1970, - /* 1970 */ 625, 1972, 1973, 620, 94, 615, 90, 10, 1402, 2006, - /* 1980 */ 166, 178, 1456, 1434, 614, 628, 1454, 367, 626, 632, - /* 1990 */ 1478, 1969, 1453, 34, 8, 13, 1477, 22, 1987, 635, - /* 2000 */ 1426, 1488, 23, 1221, 638, 1244, 622, 630, 624, 1241, - /* 2010 */ 633, 1937, 636, 621, 1238, 1232, 639, 641, 1230, 642, - /* 2020 */ 1236, 648, 1987, 95, 1235, 1234, 96, 279, 1253, 1233, - /* 2030 */ 622, 69, 1249, 1169, 1135, 1937, 658, 621, 1968, 1168, - /* 2040 */ 1167, 1166, 2003, 1165, 1164, 306, 1970, 625, 1972, 1973, - /* 2050 */ 620, 1162, 615, 1969, 1160, 1159, 1158, 1188, 670, 280, - /* 2060 */ 1156, 1155, 1968, 1154, 1153, 1152, 2003, 1151, 1150, 310, - /* 2070 */ 1970, 625, 1972, 1973, 620, 1183, 615, 1185, 1147, 1146, - /* 2080 */ 1143, 1142, 1141, 1969, 1987, 1140, 1675, 690, 692, 691, - /* 2090 */ 1673, 694, 622, 1671, 696, 695, 698, 1937, 700, 621, - /* 2100 */ 1669, 699, 702, 704, 1655, 703, 706, 1093, 1635, 283, - /* 2110 */ 710, 1610, 714, 1388, 1987, 1610, 293, 713, 1610, 1610, - /* 2120 */ 1610, 1610, 622, 1610, 1968, 1610, 1610, 1937, 2003, 621, - /* 2130 */ 1610, 302, 1970, 625, 1972, 1973, 620, 1610, 615, 1610, - /* 2140 */ 1610, 1969, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2150 */ 1610, 1610, 1610, 1610, 1968, 1610, 1610, 1969, 2003, 1610, - /* 2160 */ 1610, 311, 1970, 625, 1972, 1973, 620, 1610, 615, 1610, - /* 2170 */ 1610, 1610, 1987, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2180 */ 622, 1610, 1610, 1610, 1610, 1937, 1610, 621, 1987, 1610, - /* 2190 */ 1610, 1610, 1610, 1610, 1610, 1610, 622, 1610, 1610, 1610, - /* 2200 */ 1610, 1937, 1969, 621, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2210 */ 1610, 1610, 1968, 1610, 1610, 1610, 2003, 1610, 1969, 303, - /* 2220 */ 1970, 625, 1972, 1973, 620, 1610, 615, 1610, 1968, 1610, - /* 2230 */ 1610, 1610, 2003, 1987, 1610, 312, 1970, 625, 1972, 1973, - /* 2240 */ 620, 622, 615, 1610, 1610, 1610, 1937, 1610, 621, 1987, - /* 2250 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 622, 1610, 1610, - /* 2260 */ 1610, 1610, 1937, 1610, 621, 1610, 1610, 1610, 1610, 1610, - /* 2270 */ 1610, 1610, 1610, 1968, 1610, 1610, 1610, 2003, 1610, 1610, - /* 2280 */ 304, 1970, 625, 1972, 1973, 620, 1610, 615, 1610, 1968, - /* 2290 */ 1610, 1610, 1610, 2003, 1610, 1969, 317, 1970, 625, 1972, - /* 2300 */ 1973, 620, 1610, 615, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2310 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1969, - /* 2320 */ 1610, 1610, 1610, 1610, 1610, 1610, 1987, 1610, 1610, 1610, - /* 2330 */ 1610, 1610, 1610, 1610, 622, 1610, 1610, 1610, 1610, 1937, - /* 2340 */ 1610, 621, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2350 */ 1987, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 622, 1610, - /* 2360 */ 1610, 1610, 1610, 1937, 1610, 621, 1968, 1610, 1610, 1610, - /* 2370 */ 2003, 1610, 1610, 318, 1970, 625, 1972, 1973, 620, 1610, - /* 2380 */ 615, 1969, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2390 */ 1968, 1610, 1610, 1610, 2003, 1610, 1610, 1981, 1970, 625, - /* 2400 */ 1972, 1973, 620, 1610, 615, 1610, 1610, 1610, 1610, 1610, - /* 2410 */ 1610, 1969, 1987, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2420 */ 622, 1610, 1610, 1610, 1610, 1937, 1610, 621, 1610, 1610, - /* 2430 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2440 */ 1610, 1610, 1987, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2450 */ 622, 1610, 1968, 1610, 1610, 1937, 2003, 621, 1610, 1980, - /* 2460 */ 1970, 625, 1972, 1973, 620, 1610, 615, 1610, 1610, 1969, - /* 2470 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2480 */ 1610, 1610, 1968, 1610, 1610, 1969, 2003, 1610, 1610, 1979, - /* 2490 */ 1970, 625, 1972, 1973, 620, 1610, 615, 1610, 1610, 1610, - /* 2500 */ 1987, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 622, 1610, - /* 2510 */ 1610, 1610, 1610, 1937, 1610, 621, 1987, 1610, 1610, 1610, - /* 2520 */ 1610, 1610, 1610, 1610, 622, 1610, 1610, 1610, 1610, 1937, - /* 2530 */ 1969, 621, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2540 */ 1968, 1610, 1610, 1610, 2003, 1610, 1969, 329, 1970, 625, - /* 2550 */ 1972, 1973, 620, 1610, 615, 1610, 1968, 1610, 1610, 1610, - /* 2560 */ 2003, 1987, 1610, 330, 1970, 625, 1972, 1973, 620, 622, - /* 2570 */ 615, 1610, 1610, 1610, 1937, 1610, 621, 1987, 1610, 1610, - /* 2580 */ 1610, 1610, 1610, 1610, 1610, 622, 1610, 1610, 1610, 1610, - /* 2590 */ 1937, 1610, 621, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2600 */ 1610, 1968, 1610, 1610, 1610, 2003, 1610, 1610, 326, 1970, - /* 2610 */ 625, 1972, 1973, 620, 1610, 615, 1610, 1968, 1610, 1610, - /* 2620 */ 1610, 2003, 1610, 1969, 331, 1970, 625, 1972, 1973, 620, - /* 2630 */ 1610, 615, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, - /* 2640 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1969, 1610, 1610, - /* 2650 */ 1610, 1610, 1610, 1610, 1987, 1610, 1610, 1610, 1610, 1610, - /* 2660 */ 1610, 1610, 622, 1610, 1610, 1610, 1610, 1937, 1610, 621, - /* 2670 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1987, 1610, - /* 2680 */ 1610, 1610, 1610, 1610, 1610, 1610, 622, 1610, 1610, 1610, - /* 2690 */ 1610, 1937, 1610, 621, 623, 1610, 1610, 1610, 2003, 1610, - /* 2700 */ 1610, 309, 1970, 625, 1972, 1973, 620, 1610, 615, 1610, - /* 2710 */ 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1610, 1968, 1610, - /* 2720 */ 1610, 1610, 2003, 1610, 1610, 308, 1970, 625, 1972, 1973, - /* 2730 */ 620, 1610, 615, + /* 0 */ 1941, 2206, 1798, 607, 482, 2201, 483, 1704, 491, 1811, + /* 10 */ 483, 1704, 45, 43, 1592, 1939, 619, 31, 176, 178, + /* 20 */ 380, 2205, 1441, 38, 37, 2202, 2204, 44, 42, 41, + /* 30 */ 40, 39, 1862, 1522, 139, 1439, 1466, 2042, 1875, 347, + /* 40 */ 1924, 2028, 38, 37, 613, 358, 44, 42, 41, 40, + /* 50 */ 39, 425, 2024, 2206, 1873, 38, 37, 2201, 1517, 44, + /* 60 */ 42, 41, 40, 39, 18, 686, 385, 1469, 2060, 1868, + /* 70 */ 1870, 1447, 1666, 2205, 167, 607, 646, 2202, 2203, 1776, + /* 80 */ 1154, 2010, 1153, 648, 45, 43, 2020, 2026, 361, 570, + /* 90 */ 1135, 330, 380, 2201, 1441, 220, 14, 642, 340, 181, + /* 100 */ 2138, 2139, 550, 137, 2143, 1522, 139, 1439, 2207, 182, + /* 110 */ 602, 1155, 2041, 2202, 596, 548, 2077, 546, 739, 324, + /* 120 */ 2043, 652, 2045, 2046, 647, 645, 642, 633, 2095, 1137, + /* 130 */ 1517, 1140, 1141, 1524, 1525, 631, 18, 481, 393, 1551, + /* 140 */ 486, 1710, 392, 1447, 1691, 1262, 674, 673, 672, 1266, + /* 150 */ 671, 1268, 1269, 670, 1271, 667, 176, 1277, 664, 1279, + /* 160 */ 1280, 661, 658, 1497, 1507, 1941, 607, 617, 14, 1523, + /* 170 */ 1526, 267, 2138, 606, 383, 133, 605, 371, 1925, 2201, + /* 180 */ 1938, 619, 161, 570, 1442, 618, 1440, 2201, 2010, 359, + /* 190 */ 739, 1824, 631, 270, 594, 182, 1552, 139, 1873, 2202, + /* 200 */ 596, 500, 2207, 182, 631, 1524, 1525, 2202, 596, 590, + /* 210 */ 1445, 1446, 247, 1496, 1499, 1500, 1501, 1502, 1503, 1504, + /* 220 */ 1505, 1506, 644, 640, 1515, 1516, 1518, 1519, 1520, 1521, + /* 230 */ 2, 61, 498, 92, 1934, 1497, 1507, 585, 106, 686, + /* 240 */ 122, 1523, 1526, 121, 120, 119, 118, 117, 116, 115, + /* 250 */ 114, 113, 140, 1596, 352, 166, 1442, 1680, 1440, 1466, + /* 260 */ 1814, 609, 180, 2138, 2139, 1465, 137, 2143, 48, 34, + /* 270 */ 378, 1546, 1547, 1548, 1549, 1550, 1554, 1555, 1556, 1557, + /* 280 */ 48, 61, 1445, 1446, 1222, 1496, 1499, 1500, 1501, 1502, + /* 290 */ 1503, 1504, 1505, 1506, 644, 640, 1515, 1516, 1518, 1519, + /* 300 */ 1520, 1521, 2, 2028, 11, 45, 43, 44, 42, 41, + /* 310 */ 40, 39, 1466, 380, 2024, 1441, 353, 742, 351, 350, + /* 320 */ 1224, 523, 591, 586, 579, 525, 1522, 1466, 1439, 490, + /* 330 */ 2042, 295, 486, 1710, 607, 35, 288, 38, 37, 603, + /* 340 */ 412, 44, 42, 41, 40, 39, 175, 524, 2020, 2026, + /* 350 */ 362, 1517, 732, 728, 724, 720, 293, 18, 86, 642, + /* 360 */ 488, 2060, 414, 410, 1447, 139, 484, 559, 417, 649, + /* 370 */ 416, 2145, 349, 1154, 2010, 1153, 648, 45, 43, 1527, + /* 380 */ 1467, 1817, 2206, 185, 11, 380, 9, 1441, 61, 14, + /* 390 */ 279, 280, 65, 107, 415, 278, 286, 2142, 1522, 1468, + /* 400 */ 1439, 634, 1498, 2102, 1155, 2041, 1737, 632, 1690, 2077, + /* 410 */ 632, 739, 168, 2043, 652, 2045, 2046, 647, 1669, 642, + /* 420 */ 185, 132, 678, 1517, 187, 1866, 1524, 1525, 521, 628, + /* 430 */ 183, 2138, 2139, 185, 137, 2143, 1447, 632, 1822, 122, + /* 440 */ 11, 1822, 121, 120, 119, 118, 117, 116, 115, 114, + /* 450 */ 113, 132, 2010, 571, 2167, 194, 1497, 1507, 526, 1875, + /* 460 */ 100, 46, 1523, 1526, 273, 636, 368, 2102, 1822, 272, + /* 470 */ 1360, 1361, 536, 535, 534, 1873, 1654, 1442, 2031, 1440, + /* 480 */ 136, 530, 1815, 739, 61, 529, 1404, 463, 241, 1905, + /* 490 */ 528, 533, 83, 1305, 1306, 82, 527, 237, 1524, 1525, + /* 500 */ 1869, 1870, 1467, 1445, 1446, 1661, 1496, 1499, 1500, 1501, + /* 510 */ 1502, 1503, 1504, 1505, 1506, 644, 640, 1515, 1516, 1518, + /* 520 */ 1519, 1520, 1521, 2, 536, 535, 534, 2033, 1497, 1507, + /* 530 */ 1668, 1875, 136, 530, 1523, 1526, 632, 529, 345, 632, + /* 540 */ 185, 1447, 528, 533, 269, 198, 197, 1873, 527, 1442, + /* 550 */ 54, 1440, 618, 423, 131, 130, 129, 128, 127, 126, + /* 560 */ 125, 124, 123, 1415, 1416, 419, 677, 1822, 462, 418, + /* 570 */ 1822, 41, 40, 39, 2042, 1445, 1446, 618, 1496, 1499, + /* 580 */ 1500, 1501, 1502, 1503, 1504, 1505, 1506, 644, 640, 1515, + /* 590 */ 1516, 1518, 1519, 1520, 1521, 2, 45, 43, 1468, 616, + /* 600 */ 1920, 1934, 86, 1660, 380, 2060, 1441, 632, 595, 632, + /* 610 */ 570, 190, 2201, 649, 2201, 1377, 1378, 1522, 2010, 1439, + /* 620 */ 648, 424, 221, 433, 627, 1818, 1934, 594, 182, 2207, + /* 630 */ 182, 61, 2202, 596, 2202, 596, 185, 171, 1822, 1689, + /* 640 */ 1822, 443, 1517, 517, 513, 509, 505, 218, 632, 2041, + /* 650 */ 442, 1376, 1379, 2077, 1620, 1447, 110, 2043, 652, 2045, + /* 660 */ 2046, 647, 448, 642, 49, 372, 1532, 1688, 45, 43, + /* 670 */ 2130, 2205, 1466, 164, 2129, 2126, 380, 698, 1441, 1822, + /* 680 */ 46, 541, 1824, 2010, 87, 1687, 2145, 216, 1807, 1522, + /* 690 */ 1686, 1439, 141, 38, 37, 2101, 551, 44, 42, 41, + /* 700 */ 40, 39, 739, 582, 581, 1618, 1619, 1621, 1622, 1623, + /* 710 */ 234, 2010, 2141, 1685, 1517, 38, 37, 1524, 1525, 44, + /* 720 */ 42, 41, 40, 39, 236, 544, 33, 1447, 235, 2010, + /* 730 */ 538, 1799, 38, 37, 2010, 233, 44, 42, 41, 40, + /* 740 */ 39, 1684, 595, 269, 1553, 2145, 2201, 1497, 1507, 1875, + /* 750 */ 632, 27, 14, 1523, 1526, 215, 209, 2010, 13, 12, + /* 760 */ 214, 594, 182, 496, 449, 1874, 2202, 596, 1442, 383, + /* 770 */ 1440, 2140, 69, 1589, 739, 68, 2060, 164, 1683, 207, + /* 780 */ 1809, 1822, 1631, 185, 589, 2010, 1824, 89, 335, 1524, + /* 790 */ 1525, 357, 1978, 552, 1445, 1446, 1469, 1496, 1499, 1500, + /* 800 */ 1501, 1502, 1503, 1504, 1505, 1506, 644, 640, 1515, 1516, + /* 810 */ 1518, 1519, 1520, 1521, 2, 1498, 185, 32, 1805, 1497, + /* 820 */ 1507, 333, 2010, 1464, 684, 1523, 1526, 1558, 164, 588, + /* 830 */ 456, 710, 708, 470, 1875, 152, 469, 1825, 238, 684, + /* 840 */ 1442, 373, 1440, 154, 153, 681, 680, 679, 151, 1682, + /* 850 */ 1873, 439, 1679, 471, 1565, 676, 441, 1920, 154, 153, + /* 860 */ 681, 680, 679, 151, 532, 531, 1445, 1446, 192, 1496, + /* 870 */ 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 644, 640, + /* 880 */ 1515, 1516, 1518, 1519, 1520, 1521, 2, 699, 53, 1792, + /* 890 */ 632, 682, 165, 2010, 1866, 1678, 2010, 308, 348, 1734, + /* 900 */ 632, 38, 37, 1797, 499, 44, 42, 41, 40, 39, + /* 910 */ 429, 306, 72, 242, 1819, 71, 1677, 1920, 643, 38, + /* 920 */ 37, 1822, 1875, 44, 42, 41, 40, 39, 196, 384, + /* 930 */ 51, 1822, 3, 203, 478, 476, 473, 598, 1873, 2010, + /* 940 */ 467, 191, 525, 461, 460, 459, 458, 455, 454, 453, + /* 950 */ 452, 451, 447, 446, 445, 444, 332, 436, 435, 434, + /* 960 */ 2010, 431, 430, 346, 524, 716, 715, 714, 713, 390, + /* 970 */ 61, 712, 711, 143, 706, 705, 704, 703, 702, 701, + /* 980 */ 700, 156, 696, 695, 694, 389, 388, 691, 690, 689, + /* 990 */ 688, 687, 632, 632, 1676, 38, 37, 565, 2003, 44, + /* 1000 */ 42, 41, 40, 39, 1441, 632, 239, 566, 1608, 108, + /* 1010 */ 2042, 684, 1996, 632, 632, 632, 1469, 1439, 1588, 629, + /* 1020 */ 8, 1675, 1674, 1822, 1822, 1673, 1672, 630, 611, 615, + /* 1030 */ 154, 153, 681, 680, 679, 151, 1822, 683, 2010, 1466, + /* 1040 */ 1866, 2060, 1671, 570, 1822, 1822, 1822, 2201, 152, 610, + /* 1050 */ 80, 79, 422, 1447, 2010, 189, 648, 632, 163, 2042, + /* 1060 */ 400, 302, 2207, 182, 1852, 2010, 2010, 2202, 596, 2010, + /* 1070 */ 2010, 283, 599, 2028, 331, 1140, 1141, 408, 1450, 406, + /* 1080 */ 402, 398, 395, 415, 2024, 2041, 2010, 1681, 1822, 2077, + /* 1090 */ 2060, 1813, 109, 2043, 652, 2045, 2046, 647, 649, 642, + /* 1100 */ 739, 1410, 2024, 2010, 179, 648, 2130, 1777, 386, 632, + /* 1110 */ 374, 2126, 2150, 1585, 426, 639, 164, 2029, 2020, 2026, + /* 1120 */ 375, 632, 185, 387, 184, 1824, 2042, 427, 2024, 642, + /* 1130 */ 2004, 145, 2156, 134, 2041, 289, 2020, 2026, 2077, 1800, + /* 1140 */ 1822, 109, 2043, 652, 2045, 2046, 647, 642, 642, 52, + /* 1150 */ 152, 142, 1822, 149, 2101, 2130, 569, 2060, 2042, 374, + /* 1160 */ 2126, 246, 2020, 2026, 73, 610, 1442, 1711, 1440, 1724, + /* 1170 */ 2010, 226, 648, 642, 224, 570, 1717, 228, 230, 2201, + /* 1180 */ 227, 229, 1498, 147, 232, 245, 554, 231, 553, 2060, + /* 1190 */ 2170, 537, 1445, 1446, 2207, 182, 1715, 649, 539, 2202, + /* 1200 */ 596, 2041, 2010, 1413, 648, 2077, 1449, 2042, 109, 2043, + /* 1210 */ 652, 2045, 2046, 647, 81, 642, 734, 63, 542, 63, + /* 1220 */ 179, 251, 2130, 1663, 1664, 90, 374, 2126, 1585, 1453, + /* 1230 */ 152, 47, 276, 2041, 70, 13, 12, 2077, 2060, 105, + /* 1240 */ 109, 2043, 652, 2045, 2046, 647, 649, 642, 2157, 102, + /* 1250 */ 264, 2010, 2221, 648, 2130, 583, 150, 557, 374, 2126, + /* 1260 */ 152, 1543, 63, 47, 47, 219, 2042, 258, 2061, 2164, + /* 1270 */ 1617, 656, 1616, 692, 253, 150, 152, 135, 1184, 150, + /* 1280 */ 391, 1929, 2041, 614, 1374, 281, 2077, 624, 1705, 109, + /* 1290 */ 2043, 652, 2045, 2046, 647, 1203, 642, 2060, 693, 600, + /* 1300 */ 2042, 2221, 570, 2130, 1863, 649, 2201, 374, 2126, 285, + /* 1310 */ 2010, 2160, 648, 1255, 1185, 1559, 1508, 301, 2177, 608, + /* 1320 */ 1201, 2207, 182, 263, 1283, 266, 2202, 596, 1287, 1294, + /* 1330 */ 1292, 2060, 155, 1, 399, 4, 394, 296, 344, 649, + /* 1340 */ 1397, 2041, 195, 428, 2010, 2077, 648, 1469, 109, 2043, + /* 1350 */ 652, 2045, 2046, 647, 432, 642, 1930, 1452, 437, 465, + /* 1360 */ 2221, 1464, 2130, 450, 1922, 457, 374, 2126, 464, 466, + /* 1370 */ 472, 474, 200, 2042, 475, 2041, 477, 577, 479, 2077, + /* 1380 */ 1470, 480, 109, 2043, 652, 2045, 2046, 647, 489, 642, + /* 1390 */ 1472, 206, 377, 376, 2221, 492, 2130, 1467, 493, 208, + /* 1400 */ 374, 2126, 1455, 1471, 2060, 494, 1473, 211, 495, 497, + /* 1410 */ 1157, 2195, 649, 1522, 518, 1448, 519, 2010, 213, 648, + /* 1420 */ 84, 85, 2042, 522, 217, 501, 520, 1987, 1812, 223, + /* 1430 */ 1808, 334, 225, 112, 1984, 1983, 556, 558, 1517, 88, + /* 1440 */ 240, 148, 157, 158, 1810, 1806, 297, 560, 2041, 159, + /* 1450 */ 160, 1447, 2077, 2060, 243, 109, 2043, 652, 2045, 2046, + /* 1460 */ 647, 649, 642, 561, 567, 584, 2010, 2221, 648, 2130, + /* 1470 */ 622, 2176, 564, 374, 2126, 2175, 574, 580, 593, 363, + /* 1480 */ 587, 2161, 575, 7, 2149, 2171, 2042, 2152, 573, 249, + /* 1490 */ 172, 257, 259, 252, 260, 572, 261, 2041, 638, 364, + /* 1500 */ 604, 2077, 2224, 1585, 109, 2043, 652, 2045, 2046, 647, + /* 1510 */ 601, 642, 138, 1468, 265, 262, 2105, 2060, 2130, 612, + /* 1520 */ 2200, 2146, 374, 2126, 367, 649, 271, 1474, 95, 1935, + /* 1530 */ 2010, 620, 648, 625, 298, 621, 1949, 1948, 1947, 299, + /* 1540 */ 370, 626, 97, 99, 1823, 60, 2111, 300, 101, 303, + /* 1550 */ 1793, 292, 735, 1867, 2042, 327, 736, 654, 738, 336, + /* 1560 */ 312, 2041, 326, 50, 1456, 2077, 1451, 316, 109, 2043, + /* 1570 */ 652, 2045, 2046, 647, 305, 642, 307, 2002, 2001, 2000, + /* 1580 */ 2103, 77, 2130, 337, 2042, 2060, 374, 2126, 1997, 396, + /* 1590 */ 1459, 1461, 397, 649, 1432, 1433, 188, 401, 2010, 1995, + /* 1600 */ 648, 405, 403, 640, 1515, 1516, 1518, 1519, 1520, 1521, + /* 1610 */ 404, 1994, 407, 1993, 2042, 2060, 409, 1992, 411, 1991, + /* 1620 */ 413, 78, 1400, 649, 1399, 1961, 1960, 1959, 2010, 2041, + /* 1630 */ 648, 420, 421, 2077, 1958, 1957, 109, 2043, 652, 2045, + /* 1640 */ 2046, 647, 1351, 642, 2042, 2060, 1913, 1912, 635, 1910, + /* 1650 */ 2130, 144, 1909, 649, 374, 2126, 1908, 1911, 2010, 2041, + /* 1660 */ 648, 1907, 1906, 2077, 193, 438, 110, 2043, 652, 2045, + /* 1670 */ 2046, 647, 1904, 642, 1903, 2060, 1902, 1901, 2042, 440, + /* 1680 */ 2130, 1915, 1900, 649, 637, 2126, 1899, 1898, 2010, 650, + /* 1690 */ 648, 1897, 1896, 2077, 1895, 1894, 110, 2043, 652, 2045, + /* 1700 */ 2046, 647, 1893, 642, 1892, 2042, 1891, 1890, 1889, 2060, + /* 1710 */ 2130, 1888, 1887, 1886, 339, 2126, 1885, 649, 1884, 2041, + /* 1720 */ 146, 1883, 2010, 2077, 648, 1914, 169, 2043, 652, 2045, + /* 1730 */ 2046, 647, 1882, 642, 1881, 1880, 2060, 1353, 1879, 1878, + /* 1740 */ 468, 1877, 1876, 1740, 649, 1230, 199, 1739, 201, 2010, + /* 1750 */ 1738, 648, 2030, 2041, 2042, 202, 1736, 2077, 1700, 204, + /* 1760 */ 168, 2043, 652, 2045, 2046, 647, 75, 642, 177, 1143, + /* 1770 */ 485, 1142, 1699, 487, 1974, 1968, 205, 1956, 597, 2222, + /* 1780 */ 2041, 212, 1955, 76, 2077, 2060, 1933, 110, 2043, 652, + /* 1790 */ 2045, 2046, 647, 649, 642, 1801, 1177, 1735, 2010, 210, + /* 1800 */ 648, 2130, 2168, 1733, 502, 504, 2127, 503, 1731, 507, + /* 1810 */ 508, 506, 1729, 510, 1727, 511, 2042, 1714, 514, 512, + /* 1820 */ 516, 515, 1713, 1696, 1803, 62, 1299, 1298, 1802, 2041, + /* 1830 */ 1725, 222, 707, 2077, 1221, 1220, 318, 2043, 652, 2045, + /* 1840 */ 2046, 647, 1219, 642, 1218, 1215, 709, 2060, 2042, 1213, + /* 1850 */ 1214, 1212, 1718, 354, 355, 649, 540, 1716, 356, 1695, + /* 1860 */ 2010, 543, 648, 1694, 545, 1693, 549, 111, 547, 1420, + /* 1870 */ 1422, 1973, 1419, 1406, 55, 1967, 562, 1954, 1952, 2060, + /* 1880 */ 592, 1424, 2206, 26, 369, 66, 162, 649, 16, 244, + /* 1890 */ 19, 2041, 2010, 1633, 648, 2077, 576, 2042, 169, 2043, + /* 1900 */ 652, 2045, 2046, 647, 578, 642, 568, 28, 58, 248, + /* 1910 */ 563, 360, 5, 59, 2042, 250, 1615, 170, 255, 256, + /* 1920 */ 6, 254, 20, 2041, 30, 64, 1648, 2077, 2060, 2031, + /* 1930 */ 325, 2043, 652, 2045, 2046, 647, 646, 642, 29, 21, + /* 1940 */ 1607, 2010, 1653, 648, 91, 2060, 2042, 1654, 17, 1647, + /* 1950 */ 379, 2223, 365, 649, 1652, 1651, 366, 1582, 2010, 1581, + /* 1960 */ 648, 1953, 57, 268, 1951, 56, 1950, 1932, 94, 93, + /* 1970 */ 173, 2042, 2041, 274, 1931, 96, 2077, 2060, 287, 324, + /* 1980 */ 2043, 652, 2045, 2046, 647, 649, 642, 275, 2096, 2041, + /* 1990 */ 2010, 1613, 648, 2077, 102, 2042, 325, 2043, 652, 2045, + /* 2000 */ 2046, 647, 2060, 642, 22, 277, 282, 381, 623, 67, + /* 2010 */ 649, 12, 23, 1457, 1544, 2010, 1534, 648, 174, 284, + /* 2020 */ 2042, 555, 1512, 98, 1533, 2077, 2060, 10, 320, 2043, + /* 2030 */ 652, 2045, 2046, 647, 649, 642, 2080, 641, 36, 2010, + /* 2040 */ 1510, 648, 1509, 1481, 15, 24, 2041, 186, 1489, 25, + /* 2050 */ 2077, 2060, 655, 325, 2043, 652, 2045, 2046, 647, 649, + /* 2060 */ 642, 651, 653, 382, 2010, 657, 648, 1284, 659, 660, + /* 2070 */ 2041, 662, 1281, 1278, 2077, 663, 665, 309, 2043, 652, + /* 2080 */ 2045, 2046, 647, 2042, 642, 1272, 666, 668, 1261, 1270, + /* 2090 */ 669, 675, 290, 103, 104, 2041, 1293, 1276, 1275, 2077, + /* 2100 */ 1274, 1273, 310, 2043, 652, 2045, 2046, 647, 74, 642, + /* 2110 */ 2042, 1289, 1175, 685, 2060, 1209, 1208, 1207, 1206, 291, + /* 2120 */ 1205, 1204, 649, 1202, 1228, 1200, 1199, 2010, 1198, 648, + /* 2130 */ 697, 1196, 2042, 1195, 1194, 1193, 1192, 1191, 1190, 1225, + /* 2140 */ 1223, 2060, 1187, 1186, 1183, 1182, 1181, 1180, 1732, 649, + /* 2150 */ 717, 1730, 718, 719, 2010, 721, 648, 723, 2041, 1728, + /* 2160 */ 725, 727, 2077, 2060, 722, 311, 2043, 652, 2045, 2046, + /* 2170 */ 647, 649, 642, 1726, 726, 729, 2010, 730, 648, 1712, + /* 2180 */ 731, 733, 1132, 1692, 294, 2041, 737, 741, 1443, 2077, + /* 2190 */ 304, 740, 317, 2043, 652, 2045, 2046, 647, 1667, 642, + /* 2200 */ 1667, 1667, 1667, 1667, 1667, 1667, 1667, 2041, 1667, 1667, + /* 2210 */ 1667, 2077, 2042, 1667, 321, 2043, 652, 2045, 2046, 647, + /* 2220 */ 1667, 642, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 2042, + /* 2230 */ 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, + /* 2240 */ 1667, 1667, 1667, 2060, 1667, 1667, 2042, 1667, 1667, 1667, + /* 2250 */ 1667, 649, 1667, 1667, 1667, 1667, 2010, 1667, 648, 1667, + /* 2260 */ 2060, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 649, 1667, + /* 2270 */ 1667, 1667, 1667, 2010, 1667, 648, 1667, 2060, 2042, 1667, + /* 2280 */ 1667, 1667, 1667, 1667, 1667, 649, 1667, 2041, 1667, 1667, + /* 2290 */ 2010, 2077, 648, 1667, 313, 2043, 652, 2045, 2046, 647, + /* 2300 */ 1667, 642, 1667, 2042, 2041, 1667, 1667, 1667, 2077, 2060, + /* 2310 */ 1667, 322, 2043, 652, 2045, 2046, 647, 649, 642, 1667, + /* 2320 */ 1667, 2041, 2010, 1667, 648, 2077, 1667, 2042, 314, 2043, + /* 2330 */ 652, 2045, 2046, 647, 2060, 642, 1667, 1667, 1667, 1667, + /* 2340 */ 1667, 1667, 649, 1667, 1667, 1667, 1667, 2010, 1667, 648, + /* 2350 */ 1667, 1667, 2042, 2041, 1667, 1667, 1667, 2077, 2060, 1667, + /* 2360 */ 323, 2043, 652, 2045, 2046, 647, 649, 642, 1667, 1667, + /* 2370 */ 1667, 2010, 1667, 648, 1667, 1667, 1667, 1667, 2041, 1667, + /* 2380 */ 1667, 1667, 2077, 2060, 1667, 315, 2043, 652, 2045, 2046, + /* 2390 */ 647, 649, 642, 1667, 1667, 1667, 2010, 1667, 648, 1667, + /* 2400 */ 1667, 1667, 2041, 1667, 1667, 1667, 2077, 1667, 1667, 328, + /* 2410 */ 2043, 652, 2045, 2046, 647, 2042, 642, 1667, 1667, 1667, + /* 2420 */ 1667, 1667, 1667, 1667, 1667, 1667, 1667, 2041, 1667, 1667, + /* 2430 */ 1667, 2077, 1667, 1667, 329, 2043, 652, 2045, 2046, 647, + /* 2440 */ 1667, 642, 2042, 1667, 1667, 1667, 2060, 1667, 1667, 1667, + /* 2450 */ 1667, 1667, 1667, 1667, 649, 1667, 1667, 1667, 1667, 2010, + /* 2460 */ 1667, 648, 1667, 1667, 2042, 1667, 1667, 1667, 1667, 1667, + /* 2470 */ 1667, 1667, 1667, 2060, 1667, 1667, 1667, 1667, 1667, 1667, + /* 2480 */ 1667, 649, 1667, 1667, 1667, 1667, 2010, 1667, 648, 1667, + /* 2490 */ 2041, 1667, 1667, 1667, 2077, 2060, 1667, 2054, 2043, 652, + /* 2500 */ 2045, 2046, 647, 649, 642, 1667, 1667, 1667, 2010, 1667, + /* 2510 */ 648, 1667, 1667, 1667, 1667, 1667, 1667, 2041, 1667, 1667, + /* 2520 */ 1667, 2077, 1667, 1667, 2053, 2043, 652, 2045, 2046, 647, + /* 2530 */ 1667, 642, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 2041, + /* 2540 */ 1667, 1667, 1667, 2077, 2042, 1667, 2052, 2043, 652, 2045, + /* 2550 */ 2046, 647, 1667, 642, 1667, 1667, 1667, 1667, 1667, 1667, + /* 2560 */ 1667, 2042, 1667, 1667, 1667, 1667, 1667, 1667, 1667, 1667, + /* 2570 */ 1667, 1667, 1667, 1667, 1667, 2060, 1667, 1667, 2042, 1667, + /* 2580 */ 1667, 1667, 1667, 649, 1667, 1667, 1667, 1667, 2010, 1667, + /* 2590 */ 648, 1667, 2060, 1667, 1667, 1667, 1667, 1667, 1667, 1667, + /* 2600 */ 649, 1667, 1667, 1667, 1667, 2010, 1667, 648, 1667, 2060, + /* 2610 */ 2042, 1667, 1667, 1667, 1667, 1667, 1667, 649, 1667, 2041, + /* 2620 */ 1667, 1667, 2010, 2077, 648, 1667, 341, 2043, 652, 2045, + /* 2630 */ 2046, 647, 1667, 642, 1667, 2042, 2041, 1667, 1667, 1667, + /* 2640 */ 2077, 2060, 1667, 342, 2043, 652, 2045, 2046, 647, 649, + /* 2650 */ 642, 1667, 1667, 2041, 2010, 1667, 648, 2077, 1667, 2042, + /* 2660 */ 338, 2043, 652, 2045, 2046, 647, 2060, 642, 1667, 1667, + /* 2670 */ 1667, 1667, 1667, 1667, 649, 1667, 1667, 1667, 1667, 2010, + /* 2680 */ 1667, 648, 1667, 1667, 1667, 2041, 1667, 1667, 1667, 2077, + /* 2690 */ 2060, 1667, 343, 2043, 652, 2045, 2046, 647, 649, 642, + /* 2700 */ 1667, 1667, 1667, 2010, 1667, 648, 1667, 1667, 1667, 1667, + /* 2710 */ 650, 1667, 1667, 1667, 2077, 1667, 1667, 320, 2043, 652, + /* 2720 */ 2045, 2046, 647, 1667, 642, 1667, 1667, 1667, 1667, 1667, + /* 2730 */ 1667, 1667, 1667, 1667, 2041, 1667, 1667, 1667, 2077, 1667, + /* 2740 */ 1667, 319, 2043, 652, 2045, 2046, 647, 1667, 642, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 331, 369, 333, 334, 372, 373, 389, 327, 360, 420, - /* 10 */ 421, 357, 12, 13, 14, 327, 359, 431, 358, 371, - /* 20 */ 20, 435, 22, 360, 370, 12, 13, 14, 15, 16, - /* 30 */ 327, 335, 324, 33, 371, 35, 450, 451, 358, 379, - /* 40 */ 380, 455, 456, 372, 373, 349, 366, 335, 431, 401, - /* 50 */ 402, 371, 435, 373, 1, 2, 20, 389, 58, 371, - /* 60 */ 412, 358, 366, 63, 401, 402, 403, 450, 451, 366, - /* 70 */ 70, 20, 455, 456, 371, 412, 373, 332, 398, 20, - /* 80 */ 335, 336, 402, 12, 13, 405, 406, 407, 408, 409, - /* 90 */ 410, 20, 412, 22, 382, 95, 384, 389, 20, 431, - /* 100 */ 335, 398, 342, 435, 33, 402, 35, 347, 405, 406, - /* 110 */ 407, 408, 409, 410, 349, 412, 58, 117, 450, 451, - /* 120 */ 417, 356, 419, 455, 456, 351, 423, 424, 354, 58, - /* 130 */ 20, 366, 132, 133, 63, 95, 335, 457, 458, 431, - /* 140 */ 437, 70, 335, 435, 331, 105, 333, 334, 445, 96, - /* 150 */ 349, 18, 35, 95, 95, 97, 23, 356, 450, 451, - /* 160 */ 373, 161, 162, 455, 456, 332, 95, 366, 335, 336, - /* 170 */ 37, 38, 385, 366, 41, 388, 404, 65, 66, 67, - /* 180 */ 180, 0, 182, 132, 133, 73, 74, 70, 117, 360, - /* 190 */ 78, 350, 59, 60, 61, 83, 84, 161, 162, 358, - /* 200 */ 371, 89, 430, 132, 133, 95, 206, 207, 367, 209, - /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 95, 335, - /* 230 */ 401, 402, 161, 162, 427, 428, 429, 20, 431, 432, - /* 240 */ 20, 412, 435, 349, 14, 20, 65, 66, 67, 358, - /* 250 */ 20, 180, 127, 182, 73, 74, 131, 450, 451, 78, - /* 260 */ 366, 163, 455, 456, 83, 84, 168, 134, 8, 9, - /* 270 */ 89, 380, 12, 13, 14, 15, 16, 206, 207, 389, - /* 280 */ 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, - /* 290 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 4, - /* 300 */ 229, 12, 13, 245, 245, 172, 173, 174, 339, 20, - /* 310 */ 177, 22, 187, 188, 19, 358, 191, 339, 193, 327, - /* 320 */ 95, 431, 33, 366, 35, 435, 358, 44, 33, 196, - /* 330 */ 361, 353, 199, 365, 201, 202, 203, 204, 205, 361, - /* 340 */ 450, 451, 374, 48, 360, 455, 456, 58, 53, 350, - /* 350 */ 358, 4, 63, 58, 350, 371, 96, 358, 366, 70, - /* 360 */ 3, 62, 358, 371, 407, 373, 367, 14, 15, 16, - /* 370 */ 335, 367, 12, 13, 14, 20, 95, 20, 245, 96, - /* 380 */ 20, 327, 22, 163, 95, 401, 402, 403, 95, 94, - /* 390 */ 398, 341, 97, 33, 402, 35, 412, 405, 406, 407, - /* 400 */ 408, 409, 410, 229, 412, 355, 117, 415, 160, 417, - /* 410 */ 418, 419, 358, 0, 364, 423, 424, 382, 58, 384, - /* 420 */ 366, 132, 133, 126, 127, 371, 4, 373, 131, 0, - /* 430 */ 70, 108, 109, 110, 111, 112, 113, 114, 115, 116, - /* 440 */ 117, 118, 95, 120, 121, 122, 123, 124, 125, 106, - /* 450 */ 161, 162, 398, 96, 20, 95, 402, 0, 58, 405, - /* 460 */ 406, 407, 408, 409, 410, 43, 412, 45, 46, 180, - /* 470 */ 245, 182, 179, 419, 181, 62, 20, 117, 424, 335, - /* 480 */ 232, 24, 25, 26, 27, 28, 29, 30, 31, 32, - /* 490 */ 242, 62, 132, 133, 94, 206, 207, 97, 209, 210, - /* 500 */ 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, - /* 510 */ 221, 222, 223, 224, 225, 226, 227, 344, 345, 375, - /* 520 */ 163, 161, 162, 0, 21, 335, 245, 24, 25, 26, - /* 530 */ 27, 28, 29, 30, 31, 32, 341, 62, 245, 349, - /* 540 */ 180, 326, 182, 328, 21, 20, 327, 24, 25, 26, - /* 550 */ 27, 28, 29, 30, 31, 32, 366, 8, 9, 364, - /* 560 */ 327, 12, 13, 14, 15, 16, 206, 207, 95, 209, - /* 570 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - /* 580 */ 220, 221, 222, 223, 224, 225, 226, 227, 12, 13, - /* 590 */ 371, 358, 245, 44, 22, 79, 20, 373, 22, 366, - /* 600 */ 37, 20, 70, 22, 371, 166, 373, 35, 229, 33, - /* 610 */ 231, 35, 388, 358, 358, 416, 35, 418, 8, 9, - /* 620 */ 365, 365, 12, 13, 14, 15, 16, 39, 327, 374, - /* 630 */ 374, 398, 51, 360, 58, 402, 21, 335, 405, 406, - /* 640 */ 407, 408, 409, 410, 371, 412, 70, 206, 335, 34, - /* 650 */ 417, 36, 419, 137, 138, 335, 423, 424, 404, 12, - /* 660 */ 13, 98, 349, 100, 101, 70, 103, 20, 366, 22, - /* 670 */ 107, 95, 371, 63, 401, 402, 403, 359, 445, 366, - /* 680 */ 33, 175, 35, 20, 430, 412, 366, 431, 14, 250, - /* 690 */ 251, 435, 129, 117, 20, 254, 255, 256, 257, 258, - /* 700 */ 259, 260, 393, 197, 198, 58, 450, 451, 132, 133, - /* 710 */ 327, 455, 456, 126, 104, 431, 327, 70, 245, 435, - /* 720 */ 8, 9, 335, 359, 12, 13, 14, 15, 16, 427, - /* 730 */ 428, 429, 20, 431, 432, 451, 349, 161, 162, 455, - /* 740 */ 456, 431, 95, 404, 14, 435, 426, 427, 428, 429, - /* 750 */ 20, 431, 432, 366, 371, 359, 180, 416, 182, 418, - /* 760 */ 371, 451, 107, 327, 117, 455, 456, 8, 9, 430, - /* 770 */ 160, 12, 13, 14, 15, 16, 189, 190, 415, 132, - /* 780 */ 133, 418, 206, 207, 129, 209, 210, 211, 212, 213, - /* 790 */ 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, - /* 800 */ 224, 225, 226, 227, 18, 3, 20, 371, 161, 162, - /* 810 */ 20, 359, 22, 27, 344, 345, 30, 8, 9, 33, - /* 820 */ 366, 12, 13, 14, 15, 16, 163, 180, 0, 182, - /* 830 */ 368, 377, 244, 371, 48, 161, 50, 327, 4, 53, - /* 840 */ 327, 51, 232, 233, 234, 235, 236, 237, 238, 239, - /* 850 */ 240, 241, 242, 206, 207, 96, 209, 210, 211, 212, - /* 860 */ 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, - /* 870 */ 223, 224, 225, 226, 227, 163, 2, 327, 359, 327, - /* 880 */ 94, 371, 8, 9, 371, 359, 12, 13, 14, 15, - /* 890 */ 16, 161, 106, 65, 66, 67, 68, 69, 327, 71, - /* 900 */ 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, - /* 910 */ 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, - /* 920 */ 327, 371, 136, 371, 327, 139, 140, 141, 142, 143, - /* 930 */ 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, - /* 940 */ 154, 155, 371, 157, 158, 159, 327, 327, 327, 366, - /* 950 */ 366, 351, 12, 13, 354, 358, 327, 335, 335, 20, - /* 960 */ 377, 377, 22, 366, 371, 335, 335, 327, 371, 0, - /* 970 */ 373, 349, 349, 33, 358, 35, 20, 335, 358, 349, - /* 980 */ 349, 172, 346, 367, 348, 358, 366, 328, 366, 366, - /* 990 */ 371, 371, 371, 373, 389, 398, 366, 366, 58, 402, - /* 1000 */ 371, 374, 405, 406, 407, 408, 409, 410, 366, 412, - /* 1010 */ 70, 371, 45, 46, 417, 327, 419, 347, 398, 327, - /* 1020 */ 423, 424, 402, 0, 327, 405, 406, 407, 408, 409, - /* 1030 */ 410, 434, 412, 327, 44, 2, 431, 417, 381, 419, - /* 1040 */ 435, 8, 9, 423, 424, 12, 13, 14, 15, 16, - /* 1050 */ 368, 243, 244, 371, 434, 450, 451, 117, 0, 371, - /* 1060 */ 455, 456, 368, 371, 358, 371, 264, 33, 371, 427, - /* 1070 */ 428, 429, 366, 431, 432, 337, 107, 371, 269, 373, - /* 1080 */ 246, 0, 48, 0, 42, 42, 44, 44, 54, 55, - /* 1090 */ 56, 57, 58, 106, 459, 126, 127, 128, 129, 130, - /* 1100 */ 131, 44, 163, 22, 398, 22, 48, 192, 402, 194, - /* 1110 */ 327, 405, 406, 407, 408, 409, 410, 161, 412, 335, - /* 1120 */ 180, 99, 182, 417, 102, 419, 132, 133, 94, 423, - /* 1130 */ 424, 97, 35, 349, 327, 13, 99, 58, 22, 102, - /* 1140 */ 434, 358, 44, 156, 44, 44, 206, 207, 107, 366, - /* 1150 */ 366, 35, 335, 0, 371, 35, 373, 35, 218, 219, - /* 1160 */ 220, 221, 222, 223, 224, 358, 349, 126, 127, 128, - /* 1170 */ 129, 130, 131, 366, 151, 22, 97, 63, 371, 327, - /* 1180 */ 373, 398, 448, 366, 99, 402, 70, 102, 405, 406, - /* 1190 */ 407, 408, 409, 410, 96, 412, 96, 96, 164, 165, - /* 1200 */ 417, 167, 419, 44, 170, 398, 423, 424, 442, 402, - /* 1210 */ 358, 0, 405, 406, 407, 408, 409, 410, 366, 412, - /* 1220 */ 186, 1, 2, 371, 417, 373, 419, 358, 337, 44, - /* 1230 */ 423, 424, 335, 117, 327, 8, 9, 335, 381, 12, - /* 1240 */ 13, 14, 15, 16, 334, 44, 349, 44, 44, 44, - /* 1250 */ 398, 349, 335, 335, 402, 96, 266, 405, 406, 407, - /* 1260 */ 408, 409, 410, 366, 412, 358, 349, 349, 366, 417, - /* 1270 */ 370, 419, 335, 366, 99, 423, 424, 102, 371, 182, - /* 1280 */ 373, 96, 389, 366, 366, 381, 349, 8, 9, 433, - /* 1290 */ 327, 12, 13, 14, 15, 16, 180, 96, 182, 96, - /* 1300 */ 96, 96, 182, 366, 44, 398, 47, 44, 44, 402, - /* 1310 */ 44, 44, 405, 406, 407, 408, 409, 410, 107, 412, - /* 1320 */ 206, 358, 206, 207, 431, 268, 419, 0, 435, 366, - /* 1330 */ 423, 424, 44, 335, 371, 327, 373, 126, 127, 128, - /* 1340 */ 129, 130, 131, 450, 451, 13, 44, 349, 455, 456, - /* 1350 */ 35, 452, 425, 247, 95, 436, 96, 400, 327, 96, - /* 1360 */ 96, 398, 96, 96, 366, 402, 358, 35, 405, 406, - /* 1370 */ 407, 408, 409, 410, 366, 412, 49, 48, 399, 371, - /* 1380 */ 178, 373, 419, 391, 96, 70, 423, 424, 42, 358, - /* 1390 */ 163, 378, 20, 381, 378, 160, 376, 366, 96, 20, - /* 1400 */ 335, 335, 371, 378, 373, 376, 398, 93, 376, 335, - /* 1410 */ 402, 343, 335, 405, 406, 407, 408, 409, 410, 335, - /* 1420 */ 412, 20, 329, 329, 20, 395, 373, 419, 341, 398, - /* 1430 */ 341, 423, 424, 402, 20, 336, 405, 406, 407, 408, - /* 1440 */ 409, 410, 411, 412, 413, 414, 327, 20, 341, 390, - /* 1450 */ 341, 336, 341, 341, 341, 52, 335, 329, 397, 371, - /* 1460 */ 358, 338, 327, 338, 195, 329, 358, 335, 358, 185, - /* 1470 */ 395, 381, 394, 339, 371, 358, 358, 358, 373, 339, - /* 1480 */ 335, 253, 358, 252, 358, 366, 441, 358, 358, 358, - /* 1490 */ 371, 358, 373, 358, 261, 441, 444, 171, 371, 443, - /* 1500 */ 263, 366, 262, 441, 440, 248, 371, 438, 373, 439, - /* 1510 */ 371, 270, 371, 371, 371, 460, 267, 398, 327, 381, - /* 1520 */ 265, 402, 244, 386, 405, 406, 407, 408, 409, 410, - /* 1530 */ 386, 412, 366, 398, 454, 453, 20, 402, 339, 20, - /* 1540 */ 405, 406, 407, 408, 409, 410, 327, 412, 400, 358, - /* 1550 */ 404, 335, 384, 336, 371, 386, 386, 366, 371, 371, - /* 1560 */ 371, 371, 371, 371, 373, 446, 447, 165, 339, 383, - /* 1570 */ 95, 339, 354, 366, 422, 95, 362, 358, 348, 371, - /* 1580 */ 335, 339, 447, 330, 36, 366, 392, 329, 352, 398, - /* 1590 */ 371, 396, 373, 402, 340, 352, 405, 406, 407, 408, - /* 1600 */ 409, 410, 387, 412, 352, 325, 0, 0, 327, 387, - /* 1610 */ 187, 0, 0, 42, 0, 35, 200, 398, 35, 35, - /* 1620 */ 19, 402, 35, 0, 405, 406, 407, 408, 409, 410, - /* 1630 */ 200, 412, 35, 35, 33, 200, 0, 200, 0, 358, - /* 1640 */ 449, 0, 22, 35, 363, 0, 182, 366, 35, 48, - /* 1650 */ 180, 0, 371, 0, 373, 54, 55, 56, 57, 58, - /* 1660 */ 176, 0, 175, 327, 47, 0, 0, 0, 42, 0, - /* 1670 */ 0, 0, 0, 0, 0, 0, 0, 458, 0, 398, - /* 1680 */ 151, 35, 0, 402, 151, 0, 405, 406, 407, 408, - /* 1690 */ 409, 410, 0, 412, 358, 94, 0, 0, 97, 0, - /* 1700 */ 0, 0, 366, 0, 0, 0, 0, 371, 0, 373, - /* 1710 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, - /* 1720 */ 22, 0, 0, 135, 0, 327, 35, 0, 0, 0, - /* 1730 */ 0, 130, 0, 39, 398, 42, 47, 58, 402, 58, - /* 1740 */ 0, 405, 406, 407, 408, 409, 410, 14, 412, 47, - /* 1750 */ 414, 44, 14, 0, 0, 327, 358, 0, 0, 0, - /* 1760 */ 40, 363, 39, 171, 366, 0, 47, 0, 167, 371, - /* 1770 */ 39, 373, 0, 64, 35, 39, 0, 35, 0, 35, - /* 1780 */ 39, 0, 48, 48, 0, 184, 358, 186, 39, 39, - /* 1790 */ 35, 363, 0, 48, 366, 48, 398, 0, 0, 371, - /* 1800 */ 402, 373, 22, 405, 406, 407, 408, 409, 410, 35, - /* 1810 */ 412, 0, 35, 327, 35, 44, 22, 35, 35, 44, - /* 1820 */ 35, 35, 35, 0, 22, 0, 398, 102, 22, 327, - /* 1830 */ 402, 0, 22, 405, 406, 407, 408, 409, 410, 50, - /* 1840 */ 412, 104, 35, 0, 358, 0, 0, 35, 35, 22, - /* 1850 */ 35, 35, 366, 20, 35, 0, 96, 371, 95, 373, - /* 1860 */ 358, 35, 0, 183, 22, 0, 0, 3, 366, 96, - /* 1870 */ 249, 249, 44, 371, 327, 373, 44, 3, 44, 44, - /* 1880 */ 249, 44, 44, 95, 398, 95, 47, 96, 402, 96, - /* 1890 */ 327, 405, 406, 407, 408, 409, 410, 96, 412, 95, - /* 1900 */ 398, 47, 35, 95, 402, 358, 163, 405, 406, 407, - /* 1910 */ 408, 409, 410, 366, 412, 163, 165, 169, 371, 95, - /* 1920 */ 373, 358, 35, 35, 163, 96, 47, 35, 35, 366, - /* 1930 */ 35, 95, 47, 243, 371, 0, 373, 96, 44, 0, - /* 1940 */ 96, 0, 0, 96, 47, 398, 95, 39, 96, 402, - /* 1950 */ 47, 95, 405, 406, 407, 408, 409, 410, 95, 412, - /* 1960 */ 44, 398, 166, 164, 96, 402, 95, 327, 405, 406, - /* 1970 */ 407, 408, 409, 410, 105, 412, 95, 2, 22, 95, - /* 1980 */ 47, 47, 96, 22, 95, 35, 96, 35, 106, 35, - /* 1990 */ 228, 327, 96, 95, 230, 95, 228, 95, 358, 35, - /* 2000 */ 96, 206, 95, 22, 35, 96, 366, 95, 208, 96, - /* 2010 */ 95, 371, 95, 373, 96, 96, 95, 35, 96, 95, - /* 2020 */ 119, 107, 358, 95, 119, 119, 95, 44, 35, 119, - /* 2030 */ 366, 95, 22, 35, 64, 371, 63, 373, 398, 35, - /* 2040 */ 35, 35, 402, 35, 35, 405, 406, 407, 408, 409, - /* 2050 */ 410, 35, 412, 327, 35, 35, 35, 70, 92, 44, - /* 2060 */ 35, 35, 398, 35, 22, 35, 402, 35, 35, 405, - /* 2070 */ 406, 407, 408, 409, 410, 35, 412, 70, 35, 35, - /* 2080 */ 35, 35, 22, 327, 358, 35, 0, 35, 39, 48, - /* 2090 */ 0, 35, 366, 0, 39, 48, 35, 371, 39, 373, - /* 2100 */ 0, 48, 35, 39, 0, 48, 35, 35, 0, 22, - /* 2110 */ 21, 461, 20, 22, 358, 461, 22, 21, 461, 461, - /* 2120 */ 461, 461, 366, 461, 398, 461, 461, 371, 402, 373, - /* 2130 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 461, - /* 2140 */ 461, 327, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2150 */ 461, 461, 461, 461, 398, 461, 461, 327, 402, 461, - /* 2160 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 461, - /* 2170 */ 461, 461, 358, 461, 461, 461, 461, 461, 461, 461, - /* 2180 */ 366, 461, 461, 461, 461, 371, 461, 373, 358, 461, - /* 2190 */ 461, 461, 461, 461, 461, 461, 366, 461, 461, 461, - /* 2200 */ 461, 371, 327, 373, 461, 461, 461, 461, 461, 461, - /* 2210 */ 461, 461, 398, 461, 461, 461, 402, 461, 327, 405, - /* 2220 */ 406, 407, 408, 409, 410, 461, 412, 461, 398, 461, - /* 2230 */ 461, 461, 402, 358, 461, 405, 406, 407, 408, 409, - /* 2240 */ 410, 366, 412, 461, 461, 461, 371, 461, 373, 358, - /* 2250 */ 461, 461, 461, 461, 461, 461, 461, 366, 461, 461, - /* 2260 */ 461, 461, 371, 461, 373, 461, 461, 461, 461, 461, - /* 2270 */ 461, 461, 461, 398, 461, 461, 461, 402, 461, 461, - /* 2280 */ 405, 406, 407, 408, 409, 410, 461, 412, 461, 398, - /* 2290 */ 461, 461, 461, 402, 461, 327, 405, 406, 407, 408, - /* 2300 */ 409, 410, 461, 412, 461, 461, 461, 461, 461, 461, - /* 2310 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 327, - /* 2320 */ 461, 461, 461, 461, 461, 461, 358, 461, 461, 461, - /* 2330 */ 461, 461, 461, 461, 366, 461, 461, 461, 461, 371, - /* 2340 */ 461, 373, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2350 */ 358, 461, 461, 461, 461, 461, 461, 461, 366, 461, - /* 2360 */ 461, 461, 461, 371, 461, 373, 398, 461, 461, 461, - /* 2370 */ 402, 461, 461, 405, 406, 407, 408, 409, 410, 461, - /* 2380 */ 412, 327, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2390 */ 398, 461, 461, 461, 402, 461, 461, 405, 406, 407, - /* 2400 */ 408, 409, 410, 461, 412, 461, 461, 461, 461, 461, - /* 2410 */ 461, 327, 358, 461, 461, 461, 461, 461, 461, 461, - /* 2420 */ 366, 461, 461, 461, 461, 371, 461, 373, 461, 461, - /* 2430 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2440 */ 461, 461, 358, 461, 461, 461, 461, 461, 461, 461, - /* 2450 */ 366, 461, 398, 461, 461, 371, 402, 373, 461, 405, - /* 2460 */ 406, 407, 408, 409, 410, 461, 412, 461, 461, 327, - /* 2470 */ 461, 461, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2480 */ 461, 461, 398, 461, 461, 327, 402, 461, 461, 405, - /* 2490 */ 406, 407, 408, 409, 410, 461, 412, 461, 461, 461, - /* 2500 */ 358, 461, 461, 461, 461, 461, 461, 461, 366, 461, - /* 2510 */ 461, 461, 461, 371, 461, 373, 358, 461, 461, 461, - /* 2520 */ 461, 461, 461, 461, 366, 461, 461, 461, 461, 371, - /* 2530 */ 327, 373, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2540 */ 398, 461, 461, 461, 402, 461, 327, 405, 406, 407, - /* 2550 */ 408, 409, 410, 461, 412, 461, 398, 461, 461, 461, - /* 2560 */ 402, 358, 461, 405, 406, 407, 408, 409, 410, 366, - /* 2570 */ 412, 461, 461, 461, 371, 461, 373, 358, 461, 461, - /* 2580 */ 461, 461, 461, 461, 461, 366, 461, 461, 461, 461, - /* 2590 */ 371, 461, 373, 461, 461, 461, 461, 461, 461, 461, - /* 2600 */ 461, 398, 461, 461, 461, 402, 461, 461, 405, 406, - /* 2610 */ 407, 408, 409, 410, 461, 412, 461, 398, 461, 461, - /* 2620 */ 461, 402, 461, 327, 405, 406, 407, 408, 409, 410, - /* 2630 */ 461, 412, 461, 461, 461, 461, 461, 461, 461, 461, - /* 2640 */ 461, 461, 461, 461, 461, 461, 461, 327, 461, 461, - /* 2650 */ 461, 461, 461, 461, 358, 461, 461, 461, 461, 461, - /* 2660 */ 461, 461, 366, 461, 461, 461, 461, 371, 461, 373, - /* 2670 */ 461, 461, 461, 461, 461, 461, 461, 461, 358, 461, - /* 2680 */ 461, 461, 461, 461, 461, 461, 366, 461, 461, 461, - /* 2690 */ 461, 371, 461, 373, 398, 461, 461, 461, 402, 461, - /* 2700 */ 461, 405, 406, 407, 408, 409, 410, 461, 412, 461, - /* 2710 */ 461, 461, 461, 461, 461, 461, 461, 461, 398, 461, - /* 2720 */ 461, 461, 402, 461, 461, 405, 406, 407, 408, 409, - /* 2730 */ 410, 461, 412, + /* 0 */ 377, 439, 0, 339, 335, 443, 337, 338, 335, 363, + /* 10 */ 337, 338, 12, 13, 14, 392, 393, 2, 362, 361, + /* 20 */ 20, 459, 22, 8, 9, 463, 464, 12, 13, 14, + /* 30 */ 15, 16, 374, 33, 370, 35, 20, 331, 362, 383, + /* 40 */ 384, 364, 8, 9, 394, 369, 12, 13, 14, 15, + /* 50 */ 16, 339, 375, 439, 378, 8, 9, 443, 58, 12, + /* 60 */ 13, 14, 15, 16, 64, 63, 373, 20, 362, 376, + /* 70 */ 377, 71, 328, 459, 346, 339, 370, 463, 464, 351, + /* 80 */ 20, 375, 22, 377, 12, 13, 409, 410, 411, 439, + /* 90 */ 4, 379, 20, 443, 22, 35, 96, 420, 64, 435, + /* 100 */ 436, 437, 21, 439, 440, 33, 370, 35, 458, 459, + /* 110 */ 44, 51, 406, 463, 464, 34, 410, 36, 118, 413, + /* 120 */ 414, 415, 416, 417, 418, 419, 420, 421, 422, 43, + /* 130 */ 58, 45, 46, 133, 134, 20, 64, 336, 394, 105, + /* 140 */ 339, 340, 398, 71, 331, 109, 110, 111, 112, 113, + /* 150 */ 114, 115, 116, 117, 118, 119, 362, 121, 122, 123, + /* 160 */ 124, 125, 126, 163, 164, 377, 339, 20, 96, 169, + /* 170 */ 170, 435, 436, 437, 354, 439, 440, 389, 384, 443, + /* 180 */ 392, 393, 362, 439, 184, 339, 186, 443, 375, 369, + /* 190 */ 118, 371, 20, 58, 458, 459, 162, 370, 378, 463, + /* 200 */ 464, 63, 458, 459, 20, 133, 134, 463, 464, 20, + /* 210 */ 210, 211, 165, 213, 214, 215, 216, 217, 218, 219, + /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + /* 230 */ 230, 96, 386, 98, 388, 163, 164, 168, 343, 63, + /* 240 */ 21, 169, 170, 24, 25, 26, 27, 28, 29, 30, + /* 250 */ 31, 32, 357, 14, 37, 330, 184, 332, 186, 20, + /* 260 */ 365, 434, 435, 436, 437, 20, 439, 440, 96, 235, + /* 270 */ 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, + /* 280 */ 96, 96, 210, 211, 35, 213, 214, 215, 216, 217, + /* 290 */ 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, + /* 300 */ 228, 229, 230, 364, 232, 12, 13, 12, 13, 14, + /* 310 */ 15, 16, 20, 20, 375, 22, 99, 19, 101, 102, + /* 320 */ 71, 104, 253, 254, 255, 108, 33, 20, 35, 336, + /* 330 */ 331, 33, 339, 340, 339, 428, 429, 8, 9, 273, + /* 340 */ 179, 12, 13, 14, 15, 16, 48, 130, 409, 410, + /* 350 */ 411, 58, 54, 55, 56, 57, 58, 64, 345, 420, + /* 360 */ 14, 362, 201, 202, 71, 370, 20, 107, 183, 370, + /* 370 */ 185, 412, 359, 20, 375, 22, 377, 12, 13, 14, + /* 380 */ 20, 368, 3, 248, 232, 20, 234, 22, 96, 96, + /* 390 */ 127, 128, 4, 95, 209, 132, 98, 438, 33, 20, + /* 400 */ 35, 424, 163, 426, 51, 406, 0, 339, 331, 410, + /* 410 */ 339, 118, 413, 414, 415, 416, 417, 418, 0, 420, + /* 420 */ 248, 353, 372, 58, 353, 375, 133, 134, 360, 131, + /* 430 */ 435, 436, 437, 248, 439, 440, 71, 339, 370, 21, + /* 440 */ 232, 370, 24, 25, 26, 27, 28, 29, 30, 31, + /* 450 */ 32, 353, 375, 454, 455, 58, 163, 164, 360, 362, + /* 460 */ 343, 96, 169, 170, 166, 424, 369, 426, 370, 171, + /* 470 */ 163, 164, 66, 67, 68, 378, 97, 184, 47, 186, + /* 480 */ 74, 75, 365, 118, 96, 79, 188, 80, 190, 0, + /* 490 */ 84, 85, 95, 133, 134, 98, 90, 127, 133, 134, + /* 500 */ 376, 377, 20, 210, 211, 176, 213, 214, 215, 216, + /* 510 */ 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 520 */ 227, 228, 229, 230, 66, 67, 68, 96, 163, 164, + /* 530 */ 0, 362, 74, 75, 169, 170, 339, 79, 369, 339, + /* 540 */ 248, 71, 84, 85, 165, 138, 139, 378, 90, 184, + /* 550 */ 353, 186, 339, 353, 24, 25, 26, 27, 28, 29, + /* 560 */ 30, 31, 32, 193, 194, 394, 107, 370, 161, 398, + /* 570 */ 370, 14, 15, 16, 331, 210, 211, 339, 213, 214, + /* 580 */ 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, + /* 590 */ 225, 226, 227, 228, 229, 230, 12, 13, 20, 386, + /* 600 */ 370, 388, 345, 274, 20, 362, 22, 339, 439, 339, + /* 610 */ 439, 381, 443, 370, 443, 133, 134, 33, 375, 35, + /* 620 */ 377, 353, 33, 353, 386, 368, 388, 458, 459, 458, + /* 630 */ 459, 96, 463, 464, 463, 464, 248, 48, 370, 331, + /* 640 */ 370, 152, 58, 54, 55, 56, 57, 58, 339, 406, + /* 650 */ 161, 169, 170, 410, 210, 71, 413, 414, 415, 416, + /* 660 */ 417, 418, 353, 420, 96, 354, 14, 331, 12, 13, + /* 670 */ 427, 3, 20, 362, 431, 432, 20, 71, 22, 370, + /* 680 */ 96, 4, 371, 375, 95, 331, 412, 98, 363, 33, + /* 690 */ 331, 35, 423, 8, 9, 426, 19, 12, 13, 14, + /* 700 */ 15, 16, 118, 259, 260, 261, 262, 263, 264, 265, + /* 710 */ 33, 375, 438, 331, 58, 8, 9, 133, 134, 12, + /* 720 */ 13, 14, 15, 16, 128, 48, 2, 71, 132, 375, + /* 730 */ 53, 0, 8, 9, 375, 58, 12, 13, 14, 15, + /* 740 */ 16, 331, 439, 165, 162, 412, 443, 163, 164, 362, + /* 750 */ 339, 44, 96, 169, 170, 166, 167, 375, 1, 2, + /* 760 */ 171, 458, 459, 174, 353, 378, 463, 464, 184, 354, + /* 770 */ 186, 438, 95, 4, 118, 98, 362, 362, 331, 190, + /* 780 */ 363, 370, 97, 248, 370, 375, 371, 191, 192, 133, + /* 790 */ 134, 195, 358, 197, 210, 211, 20, 213, 214, 215, + /* 800 */ 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, + /* 810 */ 226, 227, 228, 229, 230, 163, 248, 235, 363, 163, + /* 820 */ 164, 18, 375, 20, 108, 169, 170, 245, 362, 415, + /* 830 */ 27, 348, 349, 30, 362, 44, 33, 371, 404, 108, + /* 840 */ 184, 369, 186, 127, 128, 129, 130, 131, 132, 331, + /* 850 */ 378, 48, 331, 50, 97, 363, 53, 370, 127, 128, + /* 860 */ 129, 130, 131, 132, 348, 349, 210, 211, 381, 213, + /* 870 */ 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, + /* 880 */ 224, 225, 226, 227, 228, 229, 230, 350, 97, 352, + /* 890 */ 339, 372, 18, 375, 375, 331, 375, 23, 95, 0, + /* 900 */ 339, 8, 9, 0, 353, 12, 13, 14, 15, 16, + /* 910 */ 107, 37, 38, 363, 353, 41, 331, 370, 363, 8, + /* 920 */ 9, 370, 362, 12, 13, 14, 15, 16, 381, 369, + /* 930 */ 42, 370, 44, 59, 60, 61, 62, 269, 378, 375, + /* 940 */ 137, 165, 108, 140, 141, 142, 143, 144, 145, 146, + /* 950 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, + /* 960 */ 375, 158, 159, 160, 130, 66, 67, 68, 69, 70, + /* 970 */ 96, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 980 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 990 */ 91, 92, 339, 339, 331, 8, 9, 399, 394, 12, + /* 1000 */ 13, 14, 15, 16, 22, 339, 353, 353, 97, 135, + /* 1010 */ 331, 108, 0, 339, 339, 339, 20, 35, 249, 353, + /* 1020 */ 39, 331, 331, 370, 370, 331, 331, 353, 353, 353, + /* 1030 */ 127, 128, 129, 130, 131, 132, 370, 372, 375, 20, + /* 1040 */ 375, 362, 331, 439, 370, 370, 370, 443, 44, 370, + /* 1050 */ 176, 177, 178, 71, 375, 181, 377, 339, 165, 331, + /* 1060 */ 48, 355, 458, 459, 358, 375, 375, 463, 464, 375, + /* 1070 */ 375, 353, 44, 364, 200, 45, 46, 203, 35, 205, + /* 1080 */ 206, 207, 208, 209, 375, 406, 375, 332, 370, 410, + /* 1090 */ 362, 364, 413, 414, 415, 416, 417, 418, 370, 420, + /* 1100 */ 118, 97, 375, 375, 425, 377, 427, 351, 354, 339, + /* 1110 */ 431, 432, 246, 247, 22, 64, 362, 364, 409, 410, + /* 1120 */ 411, 339, 248, 353, 445, 371, 331, 35, 375, 420, + /* 1130 */ 394, 42, 453, 44, 406, 353, 409, 410, 410, 0, + /* 1140 */ 370, 413, 414, 415, 416, 417, 418, 420, 420, 165, + /* 1150 */ 44, 423, 370, 425, 426, 427, 172, 362, 331, 431, + /* 1160 */ 432, 165, 409, 410, 107, 370, 184, 0, 186, 0, + /* 1170 */ 375, 100, 377, 420, 103, 439, 0, 100, 100, 443, + /* 1180 */ 103, 103, 163, 44, 100, 58, 196, 103, 198, 362, + /* 1190 */ 385, 22, 210, 211, 458, 459, 0, 370, 22, 463, + /* 1200 */ 464, 406, 375, 97, 377, 410, 35, 331, 413, 414, + /* 1210 */ 415, 416, 417, 418, 157, 420, 49, 44, 22, 44, + /* 1220 */ 425, 44, 427, 133, 134, 98, 431, 432, 247, 186, + /* 1230 */ 44, 44, 44, 406, 44, 1, 2, 410, 362, 96, + /* 1240 */ 413, 414, 415, 416, 417, 418, 370, 420, 453, 106, + /* 1250 */ 467, 375, 425, 377, 427, 456, 44, 394, 431, 432, + /* 1260 */ 44, 210, 44, 44, 44, 341, 331, 450, 362, 442, + /* 1270 */ 97, 44, 97, 13, 97, 44, 44, 44, 35, 44, + /* 1280 */ 341, 385, 406, 97, 97, 97, 410, 97, 338, 413, + /* 1290 */ 414, 415, 416, 417, 418, 35, 420, 362, 13, 271, + /* 1300 */ 331, 425, 439, 427, 374, 370, 443, 431, 432, 97, + /* 1310 */ 375, 385, 377, 97, 71, 97, 97, 97, 442, 441, + /* 1320 */ 35, 458, 459, 433, 97, 460, 463, 464, 97, 97, + /* 1330 */ 97, 362, 97, 444, 48, 250, 408, 396, 407, 370, + /* 1340 */ 182, 406, 42, 382, 375, 410, 377, 20, 413, 414, + /* 1350 */ 415, 416, 417, 418, 382, 420, 385, 186, 380, 162, + /* 1360 */ 425, 20, 427, 339, 339, 382, 431, 432, 380, 380, + /* 1370 */ 339, 94, 339, 331, 347, 406, 339, 442, 339, 410, + /* 1380 */ 20, 333, 413, 414, 415, 416, 417, 418, 333, 420, + /* 1390 */ 20, 345, 12, 13, 425, 401, 427, 20, 377, 345, + /* 1400 */ 431, 432, 22, 20, 362, 340, 20, 345, 395, 340, + /* 1410 */ 52, 442, 370, 33, 342, 35, 342, 375, 345, 377, + /* 1420 */ 345, 345, 331, 362, 345, 339, 333, 375, 362, 362, + /* 1430 */ 362, 333, 362, 339, 375, 375, 199, 405, 58, 96, + /* 1440 */ 343, 403, 362, 362, 362, 362, 401, 189, 406, 362, + /* 1450 */ 362, 71, 410, 362, 343, 413, 414, 415, 416, 417, + /* 1460 */ 418, 370, 420, 400, 339, 258, 375, 425, 377, 427, + /* 1470 */ 257, 449, 377, 431, 432, 449, 375, 375, 175, 375, + /* 1480 */ 375, 385, 268, 266, 442, 385, 331, 452, 267, 390, + /* 1490 */ 449, 451, 448, 390, 447, 251, 446, 406, 118, 275, + /* 1500 */ 272, 410, 468, 247, 413, 414, 415, 416, 417, 418, + /* 1510 */ 270, 420, 370, 20, 461, 408, 425, 362, 427, 339, + /* 1520 */ 462, 412, 431, 432, 340, 370, 343, 20, 343, 388, + /* 1530 */ 375, 375, 377, 167, 390, 375, 375, 375, 375, 390, + /* 1540 */ 375, 387, 343, 343, 370, 96, 430, 358, 96, 339, + /* 1550 */ 352, 343, 36, 375, 331, 402, 334, 366, 333, 391, + /* 1560 */ 356, 406, 356, 397, 184, 410, 186, 356, 413, 414, + /* 1570 */ 415, 416, 417, 418, 344, 420, 329, 0, 0, 0, + /* 1580 */ 425, 42, 427, 391, 331, 362, 431, 432, 0, 35, + /* 1590 */ 210, 211, 204, 370, 35, 35, 35, 204, 375, 0, + /* 1600 */ 377, 204, 35, 223, 224, 225, 226, 227, 228, 229, + /* 1610 */ 35, 0, 204, 0, 331, 362, 35, 0, 22, 0, + /* 1620 */ 35, 191, 186, 370, 184, 0, 0, 0, 375, 406, + /* 1630 */ 377, 180, 179, 410, 0, 0, 413, 414, 415, 416, + /* 1640 */ 417, 418, 47, 420, 331, 362, 0, 0, 425, 0, + /* 1650 */ 427, 42, 0, 370, 431, 432, 0, 0, 375, 406, + /* 1660 */ 377, 0, 0, 410, 152, 35, 413, 414, 415, 416, + /* 1670 */ 417, 418, 0, 420, 0, 362, 0, 0, 331, 152, + /* 1680 */ 427, 0, 0, 370, 431, 432, 0, 0, 375, 406, + /* 1690 */ 377, 0, 0, 410, 0, 0, 413, 414, 415, 416, + /* 1700 */ 417, 418, 0, 420, 0, 331, 0, 0, 0, 362, + /* 1710 */ 427, 0, 0, 0, 431, 432, 0, 370, 0, 406, + /* 1720 */ 42, 0, 375, 410, 377, 0, 413, 414, 415, 416, + /* 1730 */ 417, 418, 0, 420, 0, 0, 362, 22, 0, 0, + /* 1740 */ 136, 0, 0, 0, 370, 35, 58, 0, 58, 375, + /* 1750 */ 0, 377, 47, 406, 331, 58, 0, 410, 0, 42, + /* 1760 */ 413, 414, 415, 416, 417, 418, 39, 420, 44, 14, + /* 1770 */ 47, 14, 0, 47, 0, 0, 40, 0, 465, 466, + /* 1780 */ 406, 175, 0, 39, 410, 362, 0, 413, 414, 415, + /* 1790 */ 416, 417, 418, 370, 420, 0, 65, 0, 375, 39, + /* 1800 */ 377, 427, 455, 0, 35, 39, 432, 48, 0, 48, + /* 1810 */ 39, 35, 0, 35, 0, 48, 331, 0, 35, 39, + /* 1820 */ 39, 48, 0, 0, 0, 105, 35, 22, 0, 406, + /* 1830 */ 0, 103, 44, 410, 35, 35, 413, 414, 415, 416, + /* 1840 */ 417, 418, 35, 420, 35, 35, 44, 362, 331, 22, + /* 1850 */ 35, 35, 0, 22, 22, 370, 50, 0, 22, 0, + /* 1860 */ 375, 35, 377, 0, 35, 0, 22, 20, 35, 35, + /* 1870 */ 35, 0, 35, 35, 165, 0, 22, 0, 0, 362, + /* 1880 */ 457, 97, 3, 96, 367, 96, 187, 370, 252, 167, + /* 1890 */ 44, 406, 375, 97, 377, 410, 231, 331, 413, 414, + /* 1900 */ 415, 416, 417, 418, 256, 420, 173, 96, 44, 96, + /* 1910 */ 165, 165, 172, 44, 331, 97, 97, 96, 44, 47, + /* 1920 */ 172, 96, 252, 406, 44, 3, 35, 410, 362, 47, + /* 1930 */ 413, 414, 415, 416, 417, 418, 370, 420, 96, 44, + /* 1940 */ 97, 375, 97, 377, 96, 362, 331, 97, 252, 35, + /* 1950 */ 367, 466, 35, 370, 35, 35, 35, 97, 375, 97, + /* 1960 */ 377, 0, 44, 47, 0, 246, 0, 0, 39, 96, + /* 1970 */ 47, 331, 406, 47, 0, 39, 410, 362, 47, 413, + /* 1980 */ 414, 415, 416, 417, 418, 370, 420, 97, 422, 406, + /* 1990 */ 375, 97, 377, 410, 106, 331, 413, 414, 415, 416, + /* 2000 */ 417, 418, 362, 420, 96, 96, 96, 367, 168, 96, + /* 2010 */ 370, 2, 44, 22, 210, 375, 231, 377, 47, 166, + /* 2020 */ 331, 406, 97, 96, 231, 410, 362, 233, 413, 414, + /* 2030 */ 415, 416, 417, 418, 370, 420, 96, 96, 96, 375, + /* 2040 */ 97, 377, 97, 97, 96, 96, 406, 47, 22, 96, + /* 2050 */ 410, 362, 35, 413, 414, 415, 416, 417, 418, 370, + /* 2060 */ 420, 212, 107, 35, 375, 96, 377, 97, 35, 96, + /* 2070 */ 406, 35, 97, 97, 410, 96, 35, 413, 414, 415, + /* 2080 */ 416, 417, 418, 331, 420, 97, 96, 35, 22, 97, + /* 2090 */ 96, 108, 44, 96, 96, 406, 35, 120, 120, 410, + /* 2100 */ 120, 120, 413, 414, 415, 416, 417, 418, 96, 420, + /* 2110 */ 331, 22, 65, 64, 362, 35, 35, 35, 35, 44, + /* 2120 */ 35, 35, 370, 35, 71, 35, 35, 375, 35, 377, + /* 2130 */ 93, 35, 331, 35, 35, 22, 35, 35, 35, 71, + /* 2140 */ 35, 362, 35, 35, 35, 35, 22, 35, 0, 370, + /* 2150 */ 35, 0, 48, 39, 375, 35, 377, 39, 406, 0, + /* 2160 */ 35, 39, 410, 362, 48, 413, 414, 415, 416, 417, + /* 2170 */ 418, 370, 420, 0, 48, 35, 375, 48, 377, 0, + /* 2180 */ 39, 35, 35, 0, 22, 406, 21, 20, 22, 410, + /* 2190 */ 22, 21, 413, 414, 415, 416, 417, 418, 469, 420, + /* 2200 */ 469, 469, 469, 469, 469, 469, 469, 406, 469, 469, + /* 2210 */ 469, 410, 331, 469, 413, 414, 415, 416, 417, 418, + /* 2220 */ 469, 420, 469, 469, 469, 469, 469, 469, 469, 331, + /* 2230 */ 469, 469, 469, 469, 469, 469, 469, 469, 469, 469, + /* 2240 */ 469, 469, 469, 362, 469, 469, 331, 469, 469, 469, + /* 2250 */ 469, 370, 469, 469, 469, 469, 375, 469, 377, 469, + /* 2260 */ 362, 469, 469, 469, 469, 469, 469, 469, 370, 469, + /* 2270 */ 469, 469, 469, 375, 469, 377, 469, 362, 331, 469, + /* 2280 */ 469, 469, 469, 469, 469, 370, 469, 406, 469, 469, + /* 2290 */ 375, 410, 377, 469, 413, 414, 415, 416, 417, 418, + /* 2300 */ 469, 420, 469, 331, 406, 469, 469, 469, 410, 362, + /* 2310 */ 469, 413, 414, 415, 416, 417, 418, 370, 420, 469, + /* 2320 */ 469, 406, 375, 469, 377, 410, 469, 331, 413, 414, + /* 2330 */ 415, 416, 417, 418, 362, 420, 469, 469, 469, 469, + /* 2340 */ 469, 469, 370, 469, 469, 469, 469, 375, 469, 377, + /* 2350 */ 469, 469, 331, 406, 469, 469, 469, 410, 362, 469, + /* 2360 */ 413, 414, 415, 416, 417, 418, 370, 420, 469, 469, + /* 2370 */ 469, 375, 469, 377, 469, 469, 469, 469, 406, 469, + /* 2380 */ 469, 469, 410, 362, 469, 413, 414, 415, 416, 417, + /* 2390 */ 418, 370, 420, 469, 469, 469, 375, 469, 377, 469, + /* 2400 */ 469, 469, 406, 469, 469, 469, 410, 469, 469, 413, + /* 2410 */ 414, 415, 416, 417, 418, 331, 420, 469, 469, 469, + /* 2420 */ 469, 469, 469, 469, 469, 469, 469, 406, 469, 469, + /* 2430 */ 469, 410, 469, 469, 413, 414, 415, 416, 417, 418, + /* 2440 */ 469, 420, 331, 469, 469, 469, 362, 469, 469, 469, + /* 2450 */ 469, 469, 469, 469, 370, 469, 469, 469, 469, 375, + /* 2460 */ 469, 377, 469, 469, 331, 469, 469, 469, 469, 469, + /* 2470 */ 469, 469, 469, 362, 469, 469, 469, 469, 469, 469, + /* 2480 */ 469, 370, 469, 469, 469, 469, 375, 469, 377, 469, + /* 2490 */ 406, 469, 469, 469, 410, 362, 469, 413, 414, 415, + /* 2500 */ 416, 417, 418, 370, 420, 469, 469, 469, 375, 469, + /* 2510 */ 377, 469, 469, 469, 469, 469, 469, 406, 469, 469, + /* 2520 */ 469, 410, 469, 469, 413, 414, 415, 416, 417, 418, + /* 2530 */ 469, 420, 469, 469, 469, 469, 469, 469, 469, 406, + /* 2540 */ 469, 469, 469, 410, 331, 469, 413, 414, 415, 416, + /* 2550 */ 417, 418, 469, 420, 469, 469, 469, 469, 469, 469, + /* 2560 */ 469, 331, 469, 469, 469, 469, 469, 469, 469, 469, + /* 2570 */ 469, 469, 469, 469, 469, 362, 469, 469, 331, 469, + /* 2580 */ 469, 469, 469, 370, 469, 469, 469, 469, 375, 469, + /* 2590 */ 377, 469, 362, 469, 469, 469, 469, 469, 469, 469, + /* 2600 */ 370, 469, 469, 469, 469, 375, 469, 377, 469, 362, + /* 2610 */ 331, 469, 469, 469, 469, 469, 469, 370, 469, 406, + /* 2620 */ 469, 469, 375, 410, 377, 469, 413, 414, 415, 416, + /* 2630 */ 417, 418, 469, 420, 469, 331, 406, 469, 469, 469, + /* 2640 */ 410, 362, 469, 413, 414, 415, 416, 417, 418, 370, + /* 2650 */ 420, 469, 469, 406, 375, 469, 377, 410, 469, 331, + /* 2660 */ 413, 414, 415, 416, 417, 418, 362, 420, 469, 469, + /* 2670 */ 469, 469, 469, 469, 370, 469, 469, 469, 469, 375, + /* 2680 */ 469, 377, 469, 469, 469, 406, 469, 469, 469, 410, + /* 2690 */ 362, 469, 413, 414, 415, 416, 417, 418, 370, 420, + /* 2700 */ 469, 469, 469, 375, 469, 377, 469, 469, 469, 469, + /* 2710 */ 406, 469, 469, 469, 410, 469, 469, 413, 414, 415, + /* 2720 */ 416, 417, 418, 469, 420, 469, 469, 469, 469, 469, + /* 2730 */ 469, 469, 469, 469, 406, 469, 469, 469, 410, 469, + /* 2740 */ 469, 413, 414, 415, 416, 417, 418, 469, 420, }; -#define YY_SHIFT_COUNT (715) +#define YY_SHIFT_COUNT (742) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2108) +#define YY_SHIFT_MAX (2183) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 133, 0, 71, 0, 289, 289, 289, 289, 289, 289, - /* 10 */ 289, 289, 289, 360, 576, 576, 647, 576, 576, 576, - /* 20 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, - /* 30 */ 576, 576, 576, 576, 576, 576, 576, 576, 576, 576, - /* 40 */ 576, 576, 576, 576, 576, 576, 59, 225, 110, 293, - /* 50 */ 58, 281, 473, 281, 110, 110, 940, 940, 940, 281, - /* 60 */ 940, 940, 347, 281, 78, 78, 422, 422, 36, 51, - /* 70 */ 230, 230, 78, 78, 78, 78, 78, 78, 78, 217, - /* 80 */ 78, 78, 299, 78, 78, 355, 78, 434, 78, 217, - /* 90 */ 456, 78, 78, 456, 78, 456, 456, 456, 78, 475, - /* 100 */ 786, 610, 610, 503, 112, 1116, 1116, 1116, 1116, 1116, - /* 110 */ 1116, 1116, 1116, 1116, 1116, 1116, 1116, 1116, 1116, 1116, - /* 120 */ 1116, 1116, 1116, 1116, 563, 357, 36, 51, 413, 117, - /* 130 */ 220, 220, 220, 429, 379, 379, 117, 525, 525, 525, - /* 140 */ 343, 434, 174, 456, 532, 456, 532, 532, 343, 595, - /* 150 */ 323, 323, 323, 323, 323, 323, 323, 1601, 523, 181, - /* 160 */ 712, 809, 441, 581, 439, 674, 730, 790, 663, 967, - /* 170 */ 655, 939, 808, 588, 802, 808, 1042, 834, 956, 1106, - /* 180 */ 1329, 1202, 1346, 1372, 1346, 1235, 1379, 1379, 1346, 1235, - /* 190 */ 1235, 1314, 1379, 1379, 1379, 1401, 1401, 1404, 299, 434, - /* 200 */ 299, 1414, 1427, 299, 1414, 299, 299, 299, 1379, 299, - /* 210 */ 1403, 1403, 1401, 456, 456, 456, 456, 456, 456, 456, - /* 220 */ 456, 456, 456, 456, 1379, 1401, 532, 532, 532, 1269, - /* 230 */ 1404, 475, 1284, 434, 475, 1379, 1372, 1372, 532, 1228, - /* 240 */ 1231, 532, 1228, 1231, 532, 532, 456, 1233, 1326, 1228, - /* 250 */ 1237, 1240, 1257, 1106, 1241, 1249, 1255, 1278, 525, 1516, - /* 260 */ 1379, 1414, 475, 1519, 1231, 532, 532, 532, 532, 532, - /* 270 */ 1231, 532, 1402, 475, 343, 475, 525, 1475, 1480, 532, - /* 280 */ 595, 1379, 475, 1548, 1401, 2733, 2733, 2733, 2733, 2733, - /* 290 */ 2733, 2733, 2733, 2733, 828, 1034, 457, 295, 260, 549, - /* 300 */ 759, 969, 874, 1033, 1227, 1211, 1279, 1279, 1279, 1279, - /* 310 */ 1279, 1279, 1279, 1279, 1279, 1041, 125, 13, 13, 506, - /* 320 */ 400, 516, 615, 587, 297, 297, 353, 53, 248, 353, - /* 330 */ 353, 353, 283, 1058, 572, 1043, 987, 1023, 1022, 1037, - /* 340 */ 1085, 1175, 1081, 1083, 1153, 915, 1079, 1098, 1100, 1101, - /* 350 */ 994, 990, 1057, 98, 1159, 1185, 1201, 1203, 1204, 1220, - /* 360 */ 1205, 1097, 1120, 1114, 1260, 1259, 1263, 1264, 1266, 1267, - /* 370 */ 1288, 1302, 40, 1122, 1332, 1315, 1327, 1606, 1607, 1423, - /* 380 */ 1611, 1612, 1571, 1614, 1580, 1416, 1583, 1584, 1587, 1430, - /* 390 */ 1623, 1597, 1598, 1435, 1636, 1437, 1638, 1608, 1641, 1620, - /* 400 */ 1645, 1613, 1464, 1470, 1651, 1653, 1484, 1487, 1661, 1678, - /* 410 */ 1617, 1665, 1666, 1667, 1626, 1669, 1670, 1671, 1672, 1673, - /* 420 */ 1674, 1675, 1676, 1529, 1646, 1682, 1533, 1685, 1692, 1696, - /* 430 */ 1697, 1699, 1700, 1701, 1703, 1704, 1705, 1706, 1708, 1710, - /* 440 */ 1711, 1712, 1677, 1713, 1714, 1715, 1716, 1717, 1698, 1718, - /* 450 */ 1721, 1722, 1588, 1724, 1732, 1691, 1727, 1679, 1728, 1681, - /* 460 */ 1729, 1730, 1693, 1694, 1707, 1689, 1733, 1702, 1738, 1719, - /* 470 */ 1740, 1720, 1723, 1753, 1754, 1757, 1731, 1592, 1758, 1759, - /* 480 */ 1765, 1709, 1767, 1772, 1739, 1734, 1736, 1776, 1742, 1735, - /* 490 */ 1741, 1778, 1744, 1745, 1749, 1781, 1755, 1747, 1750, 1784, - /* 500 */ 1792, 1797, 1798, 1737, 1725, 1774, 1780, 1811, 1777, 1779, - /* 510 */ 1782, 1783, 1771, 1775, 1785, 1786, 1794, 1787, 1823, 1802, - /* 520 */ 1825, 1806, 1789, 1831, 1810, 1807, 1843, 1812, 1845, 1813, - /* 530 */ 1846, 1827, 1833, 1815, 1816, 1819, 1760, 1763, 1855, 1743, - /* 540 */ 1826, 1862, 1680, 1842, 1752, 1751, 1865, 1866, 1761, 1748, - /* 550 */ 1864, 1828, 1621, 1788, 1773, 1790, 1791, 1832, 1834, 1793, - /* 560 */ 1804, 1808, 1824, 1801, 1835, 1839, 1854, 1836, 1837, 1622, - /* 570 */ 1829, 1841, 1874, 1838, 1631, 1867, 1887, 1888, 1892, 1893, - /* 580 */ 1895, 1844, 1847, 1879, 1690, 1894, 1885, 1935, 1939, 1941, - /* 590 */ 1689, 1897, 1851, 1852, 1868, 1856, 1863, 1796, 1871, 1942, - /* 600 */ 1908, 1799, 1881, 1869, 1689, 1903, 1916, 1762, 1764, 1768, - /* 610 */ 1975, 1956, 1795, 1884, 1886, 1889, 1890, 1898, 1896, 1933, - /* 620 */ 1900, 1902, 1934, 1904, 1961, 1800, 1907, 1882, 1909, 1950, - /* 630 */ 1952, 1912, 1913, 1954, 1915, 1918, 1964, 1917, 1919, 1969, - /* 640 */ 1921, 1922, 1982, 1924, 1901, 1905, 1906, 1910, 1981, 1914, - /* 650 */ 1928, 1983, 1931, 1993, 1936, 1983, 1983, 2010, 1970, 1973, - /* 660 */ 1998, 2004, 2005, 2006, 2008, 2009, 2016, 2019, 2020, 2021, - /* 670 */ 1987, 1966, 2015, 2025, 2026, 2028, 2042, 2030, 2032, 2033, - /* 680 */ 2007, 1771, 2040, 1775, 2043, 2044, 2045, 2046, 2060, 2050, - /* 690 */ 2086, 2052, 2041, 2049, 2090, 2056, 2047, 2055, 2093, 2061, - /* 700 */ 2053, 2059, 2100, 2067, 2057, 2064, 2104, 2071, 2072, 2108, - /* 710 */ 2087, 2089, 2091, 2094, 2096, 2092, + /* 0 */ 874, 0, 72, 0, 293, 293, 293, 293, 293, 293, + /* 10 */ 293, 293, 293, 293, 293, 365, 584, 584, 656, 584, + /* 20 */ 584, 584, 584, 584, 584, 584, 584, 584, 584, 584, + /* 30 */ 584, 584, 584, 584, 584, 584, 584, 584, 584, 584, + /* 40 */ 584, 584, 584, 584, 584, 584, 584, 584, 172, 292, + /* 50 */ 185, 184, 135, 535, 568, 535, 184, 184, 1380, 1380, + /* 60 */ 1380, 535, 1380, 1380, 388, 535, 16, 482, 115, 115, + /* 70 */ 482, 86, 86, 307, 360, 346, 346, 115, 115, 115, + /* 80 */ 115, 115, 115, 115, 147, 115, 115, 138, 16, 115, + /* 90 */ 115, 189, 115, 16, 115, 147, 115, 147, 16, 115, + /* 100 */ 115, 16, 115, 16, 16, 16, 115, 176, 803, 34, + /* 110 */ 34, 219, 458, 982, 982, 982, 982, 982, 982, 982, + /* 120 */ 982, 982, 982, 982, 982, 982, 982, 982, 982, 982, + /* 130 */ 982, 982, 217, 379, 307, 360, 249, 578, 578, 578, + /* 140 */ 2, 152, 152, 249, 245, 245, 245, 138, 260, 208, + /* 150 */ 16, 470, 16, 470, 470, 459, 606, 36, 36, 36, + /* 160 */ 36, 36, 36, 36, 36, 298, 418, 406, 47, 329, + /* 170 */ 444, 60, 69, 239, 652, 353, 776, 1030, 834, 996, + /* 180 */ 866, 981, 668, 866, 888, 769, 1019, 1085, 1286, 1158, + /* 190 */ 1300, 1327, 1300, 1197, 1341, 1341, 1300, 1197, 1197, 1341, + /* 200 */ 1277, 1341, 1341, 1341, 1360, 1360, 1370, 138, 1377, 138, + /* 210 */ 1383, 1386, 138, 1383, 138, 138, 138, 1341, 138, 1358, + /* 220 */ 1358, 1360, 16, 16, 16, 16, 16, 16, 16, 16, + /* 230 */ 16, 16, 16, 1341, 1360, 470, 470, 470, 1237, 1343, + /* 240 */ 1370, 176, 1258, 1377, 176, 1341, 1327, 1327, 470, 1207, + /* 250 */ 1213, 470, 1207, 1213, 470, 470, 16, 1217, 1303, 1207, + /* 260 */ 1214, 1221, 1244, 1085, 1224, 1228, 1240, 1256, 245, 1493, + /* 270 */ 1341, 1383, 176, 176, 1507, 1213, 470, 470, 470, 470, + /* 280 */ 470, 1213, 470, 1366, 176, 459, 176, 245, 1449, 1452, + /* 290 */ 470, 606, 1341, 176, 1516, 1360, 2749, 2749, 2749, 2749, + /* 300 */ 2749, 2749, 2749, 2749, 2749, 899, 589, 530, 677, 685, + /* 310 */ 707, 911, 731, 15, 724, 893, 903, 987, 987, 987, + /* 320 */ 987, 987, 987, 987, 987, 987, 716, 596, 295, 295, + /* 330 */ 407, 161, 489, 397, 81, 370, 263, 263, 557, 757, + /* 340 */ 582, 557, 557, 557, 1012, 791, 1092, 1089, 1057, 1139, + /* 350 */ 1071, 1077, 1078, 1084, 1169, 1176, 1196, 990, 1004, 1106, + /* 360 */ 1127, 1173, 1175, 1177, 1090, 1028, 66, 984, 1186, 1187, + /* 370 */ 1188, 1190, 1212, 1216, 1234, 1218, 1043, 1171, 1051, 1219, + /* 380 */ 431, 1220, 1227, 1231, 1232, 1233, 1235, 1143, 1260, 1285, + /* 390 */ 1243, 1167, 1577, 1578, 1579, 1539, 1588, 1554, 1388, 1559, + /* 400 */ 1560, 1561, 1393, 1599, 1567, 1575, 1397, 1611, 1408, 1613, + /* 410 */ 1581, 1617, 1596, 1619, 1585, 1430, 1436, 1440, 1625, 1626, + /* 420 */ 1627, 1451, 1453, 1634, 1635, 1595, 1646, 1647, 1649, 1609, + /* 430 */ 1652, 1656, 1657, 1661, 1662, 1672, 1674, 1676, 1512, 1630, + /* 440 */ 1677, 1527, 1681, 1682, 1686, 1687, 1691, 1692, 1694, 1695, + /* 450 */ 1702, 1704, 1706, 1707, 1708, 1711, 1712, 1713, 1678, 1716, + /* 460 */ 1718, 1721, 1725, 1732, 1734, 1715, 1735, 1738, 1739, 1604, + /* 470 */ 1741, 1742, 1743, 1688, 1710, 1747, 1690, 1750, 1697, 1756, + /* 480 */ 1758, 1717, 1727, 1724, 1705, 1755, 1723, 1757, 1726, 1772, + /* 490 */ 1736, 1744, 1774, 1775, 1777, 1760, 1606, 1782, 1786, 1795, + /* 500 */ 1731, 1797, 1803, 1769, 1759, 1766, 1808, 1776, 1761, 1771, + /* 510 */ 1812, 1778, 1767, 1780, 1814, 1783, 1773, 1781, 1817, 1822, + /* 520 */ 1823, 1824, 1720, 1728, 1791, 1805, 1828, 1799, 1800, 1807, + /* 530 */ 1809, 1788, 1802, 1810, 1815, 1827, 1816, 1830, 1831, 1852, + /* 540 */ 1832, 1806, 1857, 1836, 1826, 1859, 1829, 1863, 1833, 1865, + /* 550 */ 1844, 1847, 1834, 1835, 1837, 1784, 1787, 1871, 1709, 1789, + /* 560 */ 1838, 1875, 1699, 1854, 1745, 1722, 1877, 1878, 1746, 1733, + /* 570 */ 1879, 1846, 1636, 1811, 1796, 1813, 1740, 1665, 1748, 1648, + /* 580 */ 1818, 1864, 1869, 1819, 1821, 1825, 1842, 1843, 1874, 1872, + /* 590 */ 1882, 1848, 1880, 1670, 1845, 1850, 1922, 1895, 1696, 1891, + /* 600 */ 1914, 1917, 1919, 1920, 1921, 1860, 1862, 1916, 1719, 1918, + /* 610 */ 1923, 1961, 1964, 1966, 1967, 1873, 1929, 1705, 1926, 1908, + /* 620 */ 1890, 1894, 1909, 1910, 1840, 1913, 1974, 1936, 1853, 1927, + /* 630 */ 1888, 1705, 1931, 1968, 1785, 1794, 1793, 2009, 1991, 1804, + /* 640 */ 1940, 1925, 1941, 1943, 1942, 1945, 1971, 1948, 1949, 2000, + /* 650 */ 1946, 2026, 1849, 1953, 1955, 1970, 2017, 2028, 1969, 1975, + /* 660 */ 2033, 1973, 1976, 2036, 1979, 1988, 2041, 1990, 1992, 2052, + /* 670 */ 1994, 1977, 1978, 1980, 1981, 2066, 1983, 1997, 2048, 1998, + /* 680 */ 2061, 2012, 2048, 2048, 2089, 2047, 2049, 2080, 2081, 2082, + /* 690 */ 2083, 2085, 2086, 2088, 2090, 2091, 2093, 2053, 2037, 2075, + /* 700 */ 2096, 2098, 2099, 2113, 2101, 2102, 2103, 2068, 1788, 2105, + /* 710 */ 1802, 2107, 2108, 2109, 2110, 2124, 2112, 2148, 2115, 2104, + /* 720 */ 2114, 2151, 2120, 2116, 2118, 2159, 2125, 2126, 2122, 2173, + /* 730 */ 2140, 2129, 2141, 2179, 2146, 2147, 2183, 2162, 2165, 2166, + /* 740 */ 2168, 2170, 2167, }; -#define YY_REDUCE_COUNT (293) -#define YY_REDUCE_MIN (-414) -#define YY_REDUCE_MAX (2320) +#define YY_REDUCE_COUNT (304) +#define YY_REDUCE_MIN (-438) +#define YY_REDUCE_MAX (2328) static const short yy_reduce_ofst[] = { - /* 0 */ -292, -297, -8, 233, 597, 620, 706, 783, 807, 852, - /* 10 */ 907, 963, 1008, 1031, 1119, -320, 54, 1135, 1191, 1219, - /* 20 */ 1281, 1336, 1398, 1428, 1486, 1502, 1547, 1563, 1640, 1664, - /* 30 */ 1726, 1756, 1814, 1830, 1875, 1891, 1968, 1992, 2054, 2084, - /* 40 */ 2142, 2158, 2203, 2219, 2296, 2320, -193, 256, 320, -383, - /* 50 */ -332, -110, 605, 893, 302, 642, -337, -16, 273, -414, - /* 60 */ -352, -171, 284, 310, -235, -199, -331, -187, -340, -368, - /* 70 */ -255, -167, -304, -106, 190, 313, 387, 622, 623, -288, - /* 80 */ 631, 784, 50, 817, 897, -43, 902, -213, 917, 35, - /* 90 */ -159, 918, 937, -32, 998, -1, 255, 4, 630, -22, - /* 100 */ 144, -411, -411, 215, -240, -312, 219, 301, 383, 389, - /* 110 */ 436, 510, 513, 550, 552, 571, 593, 619, 621, 629, - /* 120 */ 640, 688, 692, 697, -346, -228, -109, -329, 195, 173, - /* 130 */ -228, 254, 339, -31, 199, 341, 470, 454, 583, 584, - /* 140 */ -226, 224, 363, 616, 462, 627, 682, 694, 600, 636, - /* 150 */ -343, 318, 364, 396, 452, 519, 526, 309, 659, 670, - /* 160 */ 657, 635, 734, 738, 766, 869, 869, 891, 857, 910, - /* 170 */ 900, 904, 856, 856, 899, 856, 927, 919, 869, 957, - /* 180 */ 979, 992, 1013, 1012, 1016, 1020, 1065, 1066, 1025, 1029, - /* 190 */ 1032, 1068, 1074, 1077, 1084, 1093, 1094, 1030, 1087, 1053, - /* 200 */ 1089, 1099, 1059, 1107, 1115, 1109, 1111, 1112, 1121, 1113, - /* 210 */ 1123, 1125, 1128, 1102, 1108, 1110, 1117, 1118, 1124, 1126, - /* 220 */ 1129, 1130, 1131, 1133, 1132, 1136, 1088, 1103, 1127, 1061, - /* 230 */ 1075, 1134, 1078, 1105, 1140, 1145, 1090, 1138, 1139, 1045, - /* 240 */ 1137, 1141, 1054, 1144, 1142, 1143, 869, 1052, 1056, 1062, - /* 250 */ 1064, 1070, 1069, 1148, 1055, 1080, 1082, 856, 1166, 1146, - /* 260 */ 1216, 1217, 1199, 1168, 1169, 1183, 1187, 1188, 1189, 1190, - /* 270 */ 1170, 1192, 1186, 1229, 1218, 1232, 1207, 1152, 1214, 1208, - /* 280 */ 1230, 1245, 1242, 1253, 1258, 1194, 1195, 1215, 1222, 1236, - /* 290 */ 1243, 1252, 1254, 1280, + /* 0 */ -256, 679, 728, 795, 827, 876, 935, 969, 1042, 1091, + /* 10 */ 1155, 1223, 243, 1253, 1283, -294, -1, 1313, 1374, 1347, + /* 20 */ 1423, 1485, 1517, 1566, 1583, 1640, 1615, 1664, 1689, 1752, + /* 30 */ 1779, 1801, 1881, 1898, 1915, 1947, 1972, 1996, 2021, 2084, + /* 40 */ 2111, 2133, 2213, 2230, 2247, 2279, 2304, 2328, -264, 169, + /* 50 */ 171, -173, -350, 604, 736, 863, -336, -5, -323, -61, + /* 60 */ 709, 303, 727, 753, -438, -386, -180, -212, 68, 98, + /* 70 */ -377, -331, -327, -344, -307, -199, -7, 71, 197, 200, + /* 80 */ 268, 270, 309, 411, -154, 551, 561, 13, -324, 653, + /* 90 */ 654, 414, 675, 97, 676, 213, 718, 238, 311, 666, + /* 100 */ 674, 472, 782, 415, 560, 754, 770, -105, -288, -93, + /* 110 */ -93, -75, -272, -187, 77, 308, 336, 354, 359, 382, + /* 120 */ 410, 447, 518, 521, 564, 585, 663, 690, 691, 694, + /* 130 */ 695, 711, -342, -41, -206, 124, 516, -41, 274, 333, + /* 140 */ 117, -23, 41, 483, 230, 487, 547, 257, 434, 269, + /* 150 */ 466, 50, 387, 519, 665, 706, 537, -354, 325, 417, + /* 160 */ 455, 492, 550, 555, 492, 598, 755, 756, 805, 783, + /* 170 */ 799, 924, 817, 906, 906, 939, 896, 950, 930, 926, + /* 180 */ 878, 878, 865, 878, 890, 889, 906, 928, 931, 941, + /* 190 */ 961, 971, 972, 978, 1024, 1025, 983, 988, 989, 1031, + /* 200 */ 1027, 1033, 1037, 1039, 1048, 1055, 994, 1046, 1021, 1054, + /* 210 */ 1065, 1013, 1062, 1069, 1073, 1075, 1076, 1086, 1079, 1072, + /* 220 */ 1074, 1093, 1061, 1066, 1067, 1068, 1070, 1080, 1081, 1082, + /* 230 */ 1083, 1087, 1088, 1094, 1098, 1052, 1059, 1060, 1032, 1038, + /* 240 */ 1045, 1097, 1063, 1095, 1111, 1125, 1096, 1100, 1101, 1022, + /* 250 */ 1099, 1102, 1026, 1103, 1104, 1105, 906, 1035, 1040, 1041, + /* 260 */ 1044, 1047, 1050, 1107, 1034, 1058, 1053, 878, 1142, 1109, + /* 270 */ 1180, 1184, 1183, 1185, 1141, 1144, 1156, 1160, 1161, 1162, + /* 280 */ 1163, 1149, 1165, 1154, 1199, 1189, 1200, 1174, 1116, 1191, + /* 290 */ 1178, 1198, 1210, 1208, 1222, 1225, 1166, 1153, 1168, 1192, + /* 300 */ 1204, 1206, 1211, 1230, 1247, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 10 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 20 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 30 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 40 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 50 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 60 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1863, 1608, - /* 70 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 80 */ 1608, 1608, 1686, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 90 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1684, - /* 100 */ 1856, 2058, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 110 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 120 */ 1608, 1608, 1608, 1608, 1608, 2070, 1608, 1608, 1686, 1608, - /* 130 */ 2070, 2070, 2070, 1684, 2030, 2030, 1608, 1608, 1608, 1608, - /* 140 */ 1793, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1793, 1608, - /* 150 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1902, 1608, 1608, - /* 160 */ 2095, 2150, 1608, 1608, 2098, 1608, 1608, 1608, 1868, 1608, - /* 170 */ 1746, 2085, 2062, 2076, 2134, 2063, 2060, 2079, 1608, 2089, - /* 180 */ 1608, 1895, 1861, 1608, 1861, 1858, 1608, 1608, 1861, 1858, - /* 190 */ 1858, 1737, 1608, 1608, 1608, 1608, 1608, 1608, 1686, 1608, - /* 200 */ 1686, 1608, 1608, 1686, 1608, 1686, 1686, 1686, 1608, 1686, - /* 210 */ 1665, 1665, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 220 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1917, - /* 230 */ 1608, 1684, 1904, 1608, 1684, 1608, 1608, 1608, 1608, 2105, - /* 240 */ 2103, 1608, 2105, 2103, 1608, 1608, 1608, 2119, 2115, 2105, - /* 250 */ 2123, 2121, 2091, 2089, 2153, 2140, 2136, 2076, 1608, 1608, - /* 260 */ 1608, 1608, 1684, 1608, 2103, 1608, 1608, 1608, 1608, 1608, - /* 270 */ 2103, 1608, 1608, 1684, 1608, 1684, 1608, 1608, 1762, 1608, - /* 280 */ 1608, 1608, 1684, 1640, 1608, 1897, 1908, 1880, 1880, 1796, - /* 290 */ 1796, 1796, 1687, 1613, 1608, 1608, 1608, 1608, 1608, 1608, - /* 300 */ 1608, 1608, 1608, 1608, 1608, 1608, 2118, 2117, 1986, 1608, - /* 310 */ 2034, 2033, 2032, 2023, 1985, 1758, 1608, 1984, 1983, 1608, - /* 320 */ 1608, 1608, 1608, 1608, 1876, 1875, 1977, 1608, 1608, 1978, - /* 330 */ 1976, 1975, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 340 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 350 */ 1608, 2137, 2141, 1608, 1608, 1608, 1608, 1608, 1608, 2059, - /* 360 */ 1608, 1608, 1608, 1608, 1608, 1959, 1608, 1608, 1608, 1608, - /* 370 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 380 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 390 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 400 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 410 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 420 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 430 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 440 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 450 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 460 */ 1608, 1608, 1608, 1608, 1645, 1964, 1608, 1608, 1608, 1608, - /* 470 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 480 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 490 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 500 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 510 */ 1608, 1608, 1725, 1724, 1608, 1608, 1608, 1608, 1608, 1608, - /* 520 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 530 */ 1608, 1608, 1608, 1608, 1608, 1608, 1968, 1608, 1608, 1608, - /* 540 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 550 */ 2133, 2092, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 560 */ 1608, 1608, 1608, 1608, 1608, 1608, 1959, 1608, 2116, 1608, - /* 570 */ 1608, 2131, 1608, 2135, 1608, 1608, 1608, 1608, 1608, 1608, - /* 580 */ 1608, 2069, 2065, 1608, 1608, 2061, 1608, 1608, 1608, 1608, - /* 590 */ 1967, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 600 */ 1608, 1608, 1608, 1608, 1958, 1608, 2020, 1608, 1608, 1608, - /* 610 */ 2054, 1608, 1608, 2005, 1608, 1608, 1608, 1608, 1608, 1608, - /* 620 */ 1608, 1608, 1608, 1968, 1608, 1971, 1608, 1608, 1608, 1608, - /* 630 */ 1608, 1790, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 640 */ 1608, 1608, 1608, 1608, 1775, 1773, 1772, 1771, 1608, 1768, - /* 650 */ 1608, 1803, 1608, 1608, 1608, 1799, 1798, 1608, 1608, 1608, - /* 660 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 670 */ 1608, 1608, 1705, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 680 */ 1608, 1697, 1608, 1696, 1608, 1608, 1608, 1608, 1608, 1608, - /* 690 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 700 */ 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, 1608, - /* 710 */ 1608, 1608, 1608, 1608, 1608, 1608, + /* 0 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 10 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 20 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 30 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 40 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 50 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 60 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 70 */ 1665, 1665, 1665, 1923, 1665, 1665, 1665, 1665, 1665, 1665, + /* 80 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1744, 1665, 1665, + /* 90 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 100 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1742, 1916, 2132, + /* 110 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 120 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 130 */ 1665, 1665, 1665, 2144, 1665, 1665, 1665, 2144, 2144, 2144, + /* 140 */ 1742, 2104, 2104, 1665, 1665, 1665, 1665, 1744, 1977, 1665, + /* 150 */ 1665, 1665, 1665, 1665, 1665, 1851, 1665, 1665, 1665, 1665, + /* 160 */ 1665, 1875, 1665, 1665, 1665, 1969, 1665, 1665, 2169, 2225, + /* 170 */ 1665, 1665, 2172, 1665, 1665, 1665, 1928, 1665, 1804, 2159, + /* 180 */ 2136, 2150, 2209, 2137, 2134, 2153, 1665, 2163, 1665, 1962, + /* 190 */ 1921, 1665, 1921, 1918, 1665, 1665, 1921, 1918, 1918, 1665, + /* 200 */ 1795, 1665, 1665, 1665, 1665, 1665, 1665, 1744, 1665, 1744, + /* 210 */ 1665, 1665, 1744, 1665, 1744, 1744, 1744, 1665, 1744, 1722, + /* 220 */ 1722, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 230 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1989, 1975, + /* 240 */ 1665, 1742, 1971, 1665, 1742, 1665, 1665, 1665, 1665, 2180, + /* 250 */ 2178, 1665, 2180, 2178, 1665, 1665, 1665, 2194, 2190, 2180, + /* 260 */ 2198, 2196, 2165, 2163, 2228, 2215, 2211, 2150, 1665, 1665, + /* 270 */ 1665, 1665, 1742, 1742, 1665, 2178, 1665, 1665, 1665, 1665, + /* 280 */ 1665, 2178, 1665, 1665, 1742, 1665, 1742, 1665, 1665, 1820, + /* 290 */ 1665, 1665, 1665, 1742, 1697, 1665, 1964, 1980, 1946, 1946, + /* 300 */ 1854, 1854, 1854, 1745, 1670, 1665, 1665, 1665, 1665, 1665, + /* 310 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 2193, 2192, 2059, + /* 320 */ 1665, 2108, 2107, 2106, 2097, 2058, 1816, 1665, 2057, 2056, + /* 330 */ 1665, 1665, 1665, 1665, 1665, 1665, 1937, 1936, 2050, 1665, + /* 340 */ 1665, 2051, 2049, 2048, 1665, 1665, 1665, 1665, 1665, 1665, + /* 350 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 360 */ 1665, 1665, 1665, 1665, 1665, 2212, 2216, 1665, 1665, 1665, + /* 370 */ 1665, 1665, 1665, 1665, 2133, 1665, 1665, 1665, 1665, 1665, + /* 380 */ 2032, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 390 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 400 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 410 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 420 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 430 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 440 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 450 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 460 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 470 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 480 */ 1665, 1665, 1665, 1702, 2037, 1665, 1665, 1665, 1665, 1665, + /* 490 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 500 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 510 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 520 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 530 */ 1665, 1783, 1782, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 540 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 550 */ 1665, 1665, 1665, 1665, 1665, 2041, 1665, 1665, 1665, 1665, + /* 560 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 570 */ 2208, 2166, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 580 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 590 */ 2032, 1665, 2191, 1665, 1665, 2206, 1665, 2210, 1665, 1665, + /* 600 */ 1665, 1665, 1665, 1665, 1665, 2143, 2139, 1665, 1665, 2135, + /* 610 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 2040, 1665, 1665, + /* 620 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 630 */ 1665, 2031, 1665, 2094, 1665, 1665, 1665, 2128, 1665, 1665, + /* 640 */ 2079, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 650 */ 2041, 1665, 2044, 1665, 1665, 1665, 1665, 1665, 1848, 1665, + /* 660 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 670 */ 1665, 1833, 1831, 1830, 1829, 1665, 1826, 1665, 1861, 1665, + /* 680 */ 1665, 1665, 1857, 1856, 1665, 1665, 1665, 1665, 1665, 1665, + /* 690 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1763, + /* 700 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1755, 1665, + /* 710 */ 1754, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 720 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 730 */ 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, 1665, + /* 740 */ 1665, 1665, 1665, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1035,6 +1044,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* USE => nothing */ 0, /* FLUSH => nothing */ 0, /* TRIM => nothing */ + 0, /* COMPACT => nothing */ 0, /* IF => nothing */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ @@ -1133,6 +1143,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* CONSUMERS => nothing */ 0, /* SUBSCRIPTIONS => nothing */ 0, /* VNODES => nothing */ + 0, /* ALIVE => nothing */ 0, /* LIKE => nothing */ 0, /* TBNAME => nothing */ 0, /* QTAGS => nothing */ @@ -1140,6 +1151,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* INDEX => nothing */ 0, /* FUNCTION => nothing */ 0, /* INTERVAL => nothing */ + 0, /* COUNT => nothing */ + 0, /* LAST_ROW => nothing */ 0, /* TOPIC => nothing */ 0, /* WITH => nothing */ 0, /* META => nothing */ @@ -1190,6 +1203,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* WEND => nothing */ 0, /* WDURATION => nothing */ 0, /* IROWTS => nothing */ + 0, /* ISFILLED => nothing */ 0, /* CAST => nothing */ 0, /* NOW => nothing */ 0, /* TODAY => nothing */ @@ -1198,10 +1212,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* SERVER_VERSION => nothing */ 0, /* SERVER_STATUS => nothing */ 0, /* CURRENT_USER => nothing */ - 0, /* COUNT => nothing */ - 0, /* LAST_ROW => nothing */ 0, /* CASE => nothing */ - 271, /* END => ABORT */ + 276, /* END => ABORT */ 0, /* WHEN => nothing */ 0, /* THEN => nothing */ 0, /* ELSE => nothing */ @@ -1225,6 +1237,8 @@ static const YYCODETYPE yyFallback[] = { 0, /* BY => nothing */ 0, /* SESSION => nothing */ 0, /* STATE_WINDOW => nothing */ + 0, /* EVENT_WINDOW => nothing */ + 0, /* START => nothing */ 0, /* SLIDING => nothing */ 0, /* FILL => nothing */ 0, /* VALUE => nothing */ @@ -1245,58 +1259,57 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 271, /* AFTER => ABORT */ - 271, /* ATTACH => ABORT */ - 271, /* BEFORE => ABORT */ - 271, /* BEGIN => ABORT */ - 271, /* BITAND => ABORT */ - 271, /* BITNOT => ABORT */ - 271, /* BITOR => ABORT */ - 271, /* BLOCKS => ABORT */ - 271, /* CHANGE => ABORT */ - 271, /* COMMA => ABORT */ - 271, /* COMPACT => ABORT */ - 271, /* CONCAT => ABORT */ - 271, /* CONFLICT => ABORT */ - 271, /* COPY => ABORT */ - 271, /* DEFERRED => ABORT */ - 271, /* DELIMITERS => ABORT */ - 271, /* DETACH => ABORT */ - 271, /* DIVIDE => ABORT */ - 271, /* DOT => ABORT */ - 271, /* EACH => ABORT */ - 271, /* FAIL => ABORT */ - 271, /* FILE => ABORT */ - 271, /* FOR => ABORT */ - 271, /* GLOB => ABORT */ - 271, /* ID => ABORT */ - 271, /* IMMEDIATE => ABORT */ - 271, /* IMPORT => ABORT */ - 271, /* INITIALLY => ABORT */ - 271, /* INSTEAD => ABORT */ - 271, /* ISNULL => ABORT */ - 271, /* KEY => ABORT */ - 271, /* MODULES => ABORT */ - 271, /* NK_BITNOT => ABORT */ - 271, /* NK_SEMI => ABORT */ - 271, /* NOTNULL => ABORT */ - 271, /* OF => ABORT */ - 271, /* PLUS => ABORT */ - 271, /* PRIVILEGE => ABORT */ - 271, /* RAISE => ABORT */ - 271, /* REPLACE => ABORT */ - 271, /* RESTRICT => ABORT */ - 271, /* ROW => ABORT */ - 271, /* SEMI => ABORT */ - 271, /* STAR => ABORT */ - 271, /* STATEMENT => ABORT */ - 271, /* STRICT => ABORT */ - 271, /* STRING => ABORT */ - 271, /* TIMES => ABORT */ - 271, /* VALUES => ABORT */ - 271, /* VARIABLE => ABORT */ - 271, /* VIEW => ABORT */ - 271, /* WAL => ABORT */ + 276, /* AFTER => ABORT */ + 276, /* ATTACH => ABORT */ + 276, /* BEFORE => ABORT */ + 276, /* BEGIN => ABORT */ + 276, /* BITAND => ABORT */ + 276, /* BITNOT => ABORT */ + 276, /* BITOR => ABORT */ + 276, /* BLOCKS => ABORT */ + 276, /* CHANGE => ABORT */ + 276, /* COMMA => ABORT */ + 276, /* CONCAT => ABORT */ + 276, /* CONFLICT => ABORT */ + 276, /* COPY => ABORT */ + 276, /* DEFERRED => ABORT */ + 276, /* DELIMITERS => ABORT */ + 276, /* DETACH => ABORT */ + 276, /* DIVIDE => ABORT */ + 276, /* DOT => ABORT */ + 276, /* EACH => ABORT */ + 276, /* FAIL => ABORT */ + 276, /* FILE => ABORT */ + 276, /* FOR => ABORT */ + 276, /* GLOB => ABORT */ + 276, /* ID => ABORT */ + 276, /* IMMEDIATE => ABORT */ + 276, /* IMPORT => ABORT */ + 276, /* INITIALLY => ABORT */ + 276, /* INSTEAD => ABORT */ + 276, /* ISNULL => ABORT */ + 276, /* KEY => ABORT */ + 276, /* MODULES => ABORT */ + 276, /* NK_BITNOT => ABORT */ + 276, /* NK_SEMI => ABORT */ + 276, /* NOTNULL => ABORT */ + 276, /* OF => ABORT */ + 276, /* PLUS => ABORT */ + 276, /* PRIVILEGE => ABORT */ + 276, /* RAISE => ABORT */ + 276, /* REPLACE => ABORT */ + 276, /* RESTRICT => ABORT */ + 276, /* ROW => ABORT */ + 276, /* SEMI => ABORT */ + 276, /* STAR => ABORT */ + 276, /* STATEMENT => ABORT */ + 276, /* STRICT => ABORT */ + 276, /* STRING => ABORT */ + 276, /* TIMES => ABORT */ + 276, /* VALUES => ABORT */ + 276, /* VARIABLE => ABORT */ + 276, /* VIEW => ABORT */ + 276, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1446,405 +1459,413 @@ static const char *const yyTokenName[] = { /* 59 */ "USE", /* 60 */ "FLUSH", /* 61 */ "TRIM", - /* 62 */ "IF", - /* 63 */ "NOT", - /* 64 */ "EXISTS", - /* 65 */ "BUFFER", - /* 66 */ "CACHEMODEL", - /* 67 */ "CACHESIZE", - /* 68 */ "COMP", - /* 69 */ "DURATION", - /* 70 */ "NK_VARIABLE", - /* 71 */ "MAXROWS", - /* 72 */ "MINROWS", - /* 73 */ "KEEP", - /* 74 */ "PAGES", - /* 75 */ "PAGESIZE", - /* 76 */ "TSDB_PAGESIZE", - /* 77 */ "PRECISION", - /* 78 */ "REPLICA", - /* 79 */ "VGROUPS", - /* 80 */ "SINGLE_STABLE", - /* 81 */ "RETENTIONS", - /* 82 */ "SCHEMALESS", - /* 83 */ "WAL_LEVEL", - /* 84 */ "WAL_FSYNC_PERIOD", - /* 85 */ "WAL_RETENTION_PERIOD", - /* 86 */ "WAL_RETENTION_SIZE", - /* 87 */ "WAL_ROLL_PERIOD", - /* 88 */ "WAL_SEGMENT_SIZE", - /* 89 */ "STT_TRIGGER", - /* 90 */ "TABLE_PREFIX", - /* 91 */ "TABLE_SUFFIX", - /* 92 */ "NK_COLON", - /* 93 */ "MAX_SPEED", - /* 94 */ "TABLE", - /* 95 */ "NK_LP", - /* 96 */ "NK_RP", - /* 97 */ "STABLE", - /* 98 */ "ADD", - /* 99 */ "COLUMN", - /* 100 */ "MODIFY", - /* 101 */ "RENAME", - /* 102 */ "TAG", - /* 103 */ "SET", - /* 104 */ "NK_EQ", - /* 105 */ "USING", - /* 106 */ "TAGS", - /* 107 */ "COMMENT", - /* 108 */ "BOOL", - /* 109 */ "TINYINT", - /* 110 */ "SMALLINT", - /* 111 */ "INT", - /* 112 */ "INTEGER", - /* 113 */ "BIGINT", - /* 114 */ "FLOAT", - /* 115 */ "DOUBLE", - /* 116 */ "BINARY", - /* 117 */ "TIMESTAMP", - /* 118 */ "NCHAR", - /* 119 */ "UNSIGNED", - /* 120 */ "JSON", - /* 121 */ "VARCHAR", - /* 122 */ "MEDIUMBLOB", - /* 123 */ "BLOB", - /* 124 */ "VARBINARY", - /* 125 */ "DECIMAL", - /* 126 */ "MAX_DELAY", - /* 127 */ "WATERMARK", - /* 128 */ "ROLLUP", - /* 129 */ "TTL", - /* 130 */ "SMA", - /* 131 */ "DELETE_MARK", - /* 132 */ "FIRST", - /* 133 */ "LAST", - /* 134 */ "SHOW", - /* 135 */ "PRIVILEGES", - /* 136 */ "DATABASES", - /* 137 */ "TABLES", - /* 138 */ "STABLES", - /* 139 */ "MNODES", - /* 140 */ "QNODES", - /* 141 */ "FUNCTIONS", - /* 142 */ "INDEXES", - /* 143 */ "ACCOUNTS", - /* 144 */ "APPS", - /* 145 */ "CONNECTIONS", - /* 146 */ "LICENCES", - /* 147 */ "GRANTS", - /* 148 */ "QUERIES", - /* 149 */ "SCORES", - /* 150 */ "TOPICS", - /* 151 */ "VARIABLES", - /* 152 */ "CLUSTER", - /* 153 */ "BNODES", - /* 154 */ "SNODES", - /* 155 */ "TRANSACTIONS", - /* 156 */ "DISTRIBUTED", - /* 157 */ "CONSUMERS", - /* 158 */ "SUBSCRIPTIONS", - /* 159 */ "VNODES", - /* 160 */ "LIKE", - /* 161 */ "TBNAME", - /* 162 */ "QTAGS", - /* 163 */ "AS", - /* 164 */ "INDEX", - /* 165 */ "FUNCTION", - /* 166 */ "INTERVAL", - /* 167 */ "TOPIC", - /* 168 */ "WITH", - /* 169 */ "META", - /* 170 */ "CONSUMER", - /* 171 */ "GROUP", - /* 172 */ "DESC", - /* 173 */ "DESCRIBE", - /* 174 */ "RESET", - /* 175 */ "QUERY", - /* 176 */ "CACHE", - /* 177 */ "EXPLAIN", - /* 178 */ "ANALYZE", - /* 179 */ "VERBOSE", - /* 180 */ "NK_BOOL", - /* 181 */ "RATIO", - /* 182 */ "NK_FLOAT", - /* 183 */ "OUTPUTTYPE", - /* 184 */ "AGGREGATE", - /* 185 */ "BUFSIZE", - /* 186 */ "STREAM", - /* 187 */ "INTO", - /* 188 */ "TRIGGER", - /* 189 */ "AT_ONCE", - /* 190 */ "WINDOW_CLOSE", - /* 191 */ "IGNORE", - /* 192 */ "EXPIRED", - /* 193 */ "FILL_HISTORY", - /* 194 */ "UPDATE", - /* 195 */ "SUBTABLE", - /* 196 */ "KILL", - /* 197 */ "CONNECTION", - /* 198 */ "TRANSACTION", - /* 199 */ "BALANCE", - /* 200 */ "VGROUP", - /* 201 */ "MERGE", - /* 202 */ "REDISTRIBUTE", - /* 203 */ "SPLIT", - /* 204 */ "DELETE", - /* 205 */ "INSERT", - /* 206 */ "NULL", - /* 207 */ "NK_QUESTION", - /* 208 */ "NK_ARROW", - /* 209 */ "ROWTS", - /* 210 */ "QSTART", - /* 211 */ "QEND", - /* 212 */ "QDURATION", - /* 213 */ "WSTART", - /* 214 */ "WEND", - /* 215 */ "WDURATION", - /* 216 */ "IROWTS", - /* 217 */ "CAST", - /* 218 */ "NOW", - /* 219 */ "TODAY", - /* 220 */ "TIMEZONE", - /* 221 */ "CLIENT_VERSION", - /* 222 */ "SERVER_VERSION", - /* 223 */ "SERVER_STATUS", - /* 224 */ "CURRENT_USER", - /* 225 */ "COUNT", - /* 226 */ "LAST_ROW", - /* 227 */ "CASE", - /* 228 */ "END", - /* 229 */ "WHEN", - /* 230 */ "THEN", - /* 231 */ "ELSE", - /* 232 */ "BETWEEN", - /* 233 */ "IS", - /* 234 */ "NK_LT", - /* 235 */ "NK_GT", - /* 236 */ "NK_LE", - /* 237 */ "NK_GE", - /* 238 */ "NK_NE", - /* 239 */ "MATCH", - /* 240 */ "NMATCH", - /* 241 */ "CONTAINS", - /* 242 */ "IN", - /* 243 */ "JOIN", - /* 244 */ "INNER", - /* 245 */ "SELECT", - /* 246 */ "DISTINCT", - /* 247 */ "WHERE", - /* 248 */ "PARTITION", - /* 249 */ "BY", - /* 250 */ "SESSION", - /* 251 */ "STATE_WINDOW", - /* 252 */ "SLIDING", - /* 253 */ "FILL", - /* 254 */ "VALUE", - /* 255 */ "VALUE_F", - /* 256 */ "NONE", - /* 257 */ "PREV", - /* 258 */ "NULL_F", - /* 259 */ "LINEAR", - /* 260 */ "NEXT", - /* 261 */ "HAVING", - /* 262 */ "RANGE", - /* 263 */ "EVERY", - /* 264 */ "ORDER", - /* 265 */ "SLIMIT", - /* 266 */ "SOFFSET", - /* 267 */ "LIMIT", - /* 268 */ "OFFSET", - /* 269 */ "ASC", - /* 270 */ "NULLS", - /* 271 */ "ABORT", - /* 272 */ "AFTER", - /* 273 */ "ATTACH", - /* 274 */ "BEFORE", - /* 275 */ "BEGIN", - /* 276 */ "BITAND", - /* 277 */ "BITNOT", - /* 278 */ "BITOR", - /* 279 */ "BLOCKS", - /* 280 */ "CHANGE", - /* 281 */ "COMMA", - /* 282 */ "COMPACT", - /* 283 */ "CONCAT", - /* 284 */ "CONFLICT", - /* 285 */ "COPY", - /* 286 */ "DEFERRED", - /* 287 */ "DELIMITERS", - /* 288 */ "DETACH", - /* 289 */ "DIVIDE", - /* 290 */ "DOT", - /* 291 */ "EACH", - /* 292 */ "FAIL", - /* 293 */ "FILE", - /* 294 */ "FOR", - /* 295 */ "GLOB", - /* 296 */ "ID", - /* 297 */ "IMMEDIATE", - /* 298 */ "IMPORT", - /* 299 */ "INITIALLY", - /* 300 */ "INSTEAD", - /* 301 */ "ISNULL", - /* 302 */ "KEY", - /* 303 */ "MODULES", - /* 304 */ "NK_BITNOT", - /* 305 */ "NK_SEMI", - /* 306 */ "NOTNULL", - /* 307 */ "OF", - /* 308 */ "PLUS", - /* 309 */ "PRIVILEGE", - /* 310 */ "RAISE", - /* 311 */ "REPLACE", - /* 312 */ "RESTRICT", - /* 313 */ "ROW", - /* 314 */ "SEMI", - /* 315 */ "STAR", - /* 316 */ "STATEMENT", - /* 317 */ "STRICT", - /* 318 */ "STRING", - /* 319 */ "TIMES", - /* 320 */ "VALUES", - /* 321 */ "VARIABLE", - /* 322 */ "VIEW", - /* 323 */ "WAL", - /* 324 */ "cmd", - /* 325 */ "account_options", - /* 326 */ "alter_account_options", - /* 327 */ "literal", - /* 328 */ "alter_account_option", - /* 329 */ "user_name", - /* 330 */ "sysinfo_opt", - /* 331 */ "privileges", - /* 332 */ "priv_level", - /* 333 */ "priv_type_list", - /* 334 */ "priv_type", - /* 335 */ "db_name", - /* 336 */ "topic_name", - /* 337 */ "dnode_endpoint", - /* 338 */ "force_opt", - /* 339 */ "not_exists_opt", - /* 340 */ "db_options", - /* 341 */ "exists_opt", - /* 342 */ "alter_db_options", - /* 343 */ "speed_opt", - /* 344 */ "integer_list", - /* 345 */ "variable_list", - /* 346 */ "retention_list", - /* 347 */ "alter_db_option", - /* 348 */ "retention", - /* 349 */ "full_table_name", - /* 350 */ "column_def_list", - /* 351 */ "tags_def_opt", - /* 352 */ "table_options", - /* 353 */ "multi_create_clause", - /* 354 */ "tags_def", - /* 355 */ "multi_drop_clause", - /* 356 */ "alter_table_clause", - /* 357 */ "alter_table_options", - /* 358 */ "column_name", - /* 359 */ "type_name", - /* 360 */ "signed_literal", - /* 361 */ "create_subtable_clause", - /* 362 */ "specific_cols_opt", - /* 363 */ "expression_list", - /* 364 */ "drop_table_clause", - /* 365 */ "col_name_list", - /* 366 */ "table_name", - /* 367 */ "column_def", - /* 368 */ "duration_list", - /* 369 */ "rollup_func_list", - /* 370 */ "alter_table_option", - /* 371 */ "duration_literal", - /* 372 */ "rollup_func_name", - /* 373 */ "function_name", - /* 374 */ "col_name", - /* 375 */ "db_name_cond_opt", - /* 376 */ "like_pattern_opt", - /* 377 */ "table_name_cond", - /* 378 */ "from_db_opt", - /* 379 */ "tag_list_opt", - /* 380 */ "tag_item", - /* 381 */ "column_alias", - /* 382 */ "full_index_name", - /* 383 */ "index_options", - /* 384 */ "index_name", - /* 385 */ "func_list", - /* 386 */ "sliding_opt", - /* 387 */ "sma_stream_opt", - /* 388 */ "func", - /* 389 */ "query_or_subquery", - /* 390 */ "cgroup_name", - /* 391 */ "analyze_opt", - /* 392 */ "explain_options", - /* 393 */ "agg_func_opt", - /* 394 */ "bufsize_opt", - /* 395 */ "stream_name", - /* 396 */ "stream_options", - /* 397 */ "subtable_opt", - /* 398 */ "expression", - /* 399 */ "dnode_list", - /* 400 */ "where_clause_opt", - /* 401 */ "signed", - /* 402 */ "literal_func", - /* 403 */ "literal_list", - /* 404 */ "table_alias", - /* 405 */ "expr_or_subquery", - /* 406 */ "pseudo_column", - /* 407 */ "column_reference", - /* 408 */ "function_expression", - /* 409 */ "case_when_expression", - /* 410 */ "star_func", - /* 411 */ "star_func_para_list", - /* 412 */ "noarg_func", - /* 413 */ "other_para_list", - /* 414 */ "star_func_para", - /* 415 */ "when_then_list", - /* 416 */ "case_when_else_opt", - /* 417 */ "common_expression", - /* 418 */ "when_then_expr", - /* 419 */ "predicate", - /* 420 */ "compare_op", - /* 421 */ "in_op", - /* 422 */ "in_predicate_value", - /* 423 */ "boolean_value_expression", - /* 424 */ "boolean_primary", - /* 425 */ "from_clause_opt", - /* 426 */ "table_reference_list", - /* 427 */ "table_reference", - /* 428 */ "table_primary", - /* 429 */ "joined_table", - /* 430 */ "alias_opt", - /* 431 */ "subquery", - /* 432 */ "parenthesized_joined_table", - /* 433 */ "join_type", - /* 434 */ "search_condition", - /* 435 */ "query_specification", - /* 436 */ "set_quantifier_opt", - /* 437 */ "select_list", - /* 438 */ "partition_by_clause_opt", - /* 439 */ "range_opt", - /* 440 */ "every_opt", - /* 441 */ "fill_opt", - /* 442 */ "twindow_clause_opt", - /* 443 */ "group_by_clause_opt", - /* 444 */ "having_clause_opt", - /* 445 */ "select_item", - /* 446 */ "partition_list", - /* 447 */ "partition_item", - /* 448 */ "fill_mode", - /* 449 */ "group_by_list", - /* 450 */ "query_expression", - /* 451 */ "query_simple", - /* 452 */ "order_by_clause_opt", - /* 453 */ "slimit_clause_opt", - /* 454 */ "limit_clause_opt", - /* 455 */ "union_query_expression", - /* 456 */ "query_simple_or_subquery", - /* 457 */ "sort_specification_list", - /* 458 */ "sort_specification", - /* 459 */ "ordering_specification_opt", - /* 460 */ "null_ordering_opt", + /* 62 */ "COMPACT", + /* 63 */ "IF", + /* 64 */ "NOT", + /* 65 */ "EXISTS", + /* 66 */ "BUFFER", + /* 67 */ "CACHEMODEL", + /* 68 */ "CACHESIZE", + /* 69 */ "COMP", + /* 70 */ "DURATION", + /* 71 */ "NK_VARIABLE", + /* 72 */ "MAXROWS", + /* 73 */ "MINROWS", + /* 74 */ "KEEP", + /* 75 */ "PAGES", + /* 76 */ "PAGESIZE", + /* 77 */ "TSDB_PAGESIZE", + /* 78 */ "PRECISION", + /* 79 */ "REPLICA", + /* 80 */ "VGROUPS", + /* 81 */ "SINGLE_STABLE", + /* 82 */ "RETENTIONS", + /* 83 */ "SCHEMALESS", + /* 84 */ "WAL_LEVEL", + /* 85 */ "WAL_FSYNC_PERIOD", + /* 86 */ "WAL_RETENTION_PERIOD", + /* 87 */ "WAL_RETENTION_SIZE", + /* 88 */ "WAL_ROLL_PERIOD", + /* 89 */ "WAL_SEGMENT_SIZE", + /* 90 */ "STT_TRIGGER", + /* 91 */ "TABLE_PREFIX", + /* 92 */ "TABLE_SUFFIX", + /* 93 */ "NK_COLON", + /* 94 */ "MAX_SPEED", + /* 95 */ "TABLE", + /* 96 */ "NK_LP", + /* 97 */ "NK_RP", + /* 98 */ "STABLE", + /* 99 */ "ADD", + /* 100 */ "COLUMN", + /* 101 */ "MODIFY", + /* 102 */ "RENAME", + /* 103 */ "TAG", + /* 104 */ "SET", + /* 105 */ "NK_EQ", + /* 106 */ "USING", + /* 107 */ "TAGS", + /* 108 */ "COMMENT", + /* 109 */ "BOOL", + /* 110 */ "TINYINT", + /* 111 */ "SMALLINT", + /* 112 */ "INT", + /* 113 */ "INTEGER", + /* 114 */ "BIGINT", + /* 115 */ "FLOAT", + /* 116 */ "DOUBLE", + /* 117 */ "BINARY", + /* 118 */ "TIMESTAMP", + /* 119 */ "NCHAR", + /* 120 */ "UNSIGNED", + /* 121 */ "JSON", + /* 122 */ "VARCHAR", + /* 123 */ "MEDIUMBLOB", + /* 124 */ "BLOB", + /* 125 */ "VARBINARY", + /* 126 */ "DECIMAL", + /* 127 */ "MAX_DELAY", + /* 128 */ "WATERMARK", + /* 129 */ "ROLLUP", + /* 130 */ "TTL", + /* 131 */ "SMA", + /* 132 */ "DELETE_MARK", + /* 133 */ "FIRST", + /* 134 */ "LAST", + /* 135 */ "SHOW", + /* 136 */ "PRIVILEGES", + /* 137 */ "DATABASES", + /* 138 */ "TABLES", + /* 139 */ "STABLES", + /* 140 */ "MNODES", + /* 141 */ "QNODES", + /* 142 */ "FUNCTIONS", + /* 143 */ "INDEXES", + /* 144 */ "ACCOUNTS", + /* 145 */ "APPS", + /* 146 */ "CONNECTIONS", + /* 147 */ "LICENCES", + /* 148 */ "GRANTS", + /* 149 */ "QUERIES", + /* 150 */ "SCORES", + /* 151 */ "TOPICS", + /* 152 */ "VARIABLES", + /* 153 */ "CLUSTER", + /* 154 */ "BNODES", + /* 155 */ "SNODES", + /* 156 */ "TRANSACTIONS", + /* 157 */ "DISTRIBUTED", + /* 158 */ "CONSUMERS", + /* 159 */ "SUBSCRIPTIONS", + /* 160 */ "VNODES", + /* 161 */ "ALIVE", + /* 162 */ "LIKE", + /* 163 */ "TBNAME", + /* 164 */ "QTAGS", + /* 165 */ "AS", + /* 166 */ "INDEX", + /* 167 */ "FUNCTION", + /* 168 */ "INTERVAL", + /* 169 */ "COUNT", + /* 170 */ "LAST_ROW", + /* 171 */ "TOPIC", + /* 172 */ "WITH", + /* 173 */ "META", + /* 174 */ "CONSUMER", + /* 175 */ "GROUP", + /* 176 */ "DESC", + /* 177 */ "DESCRIBE", + /* 178 */ "RESET", + /* 179 */ "QUERY", + /* 180 */ "CACHE", + /* 181 */ "EXPLAIN", + /* 182 */ "ANALYZE", + /* 183 */ "VERBOSE", + /* 184 */ "NK_BOOL", + /* 185 */ "RATIO", + /* 186 */ "NK_FLOAT", + /* 187 */ "OUTPUTTYPE", + /* 188 */ "AGGREGATE", + /* 189 */ "BUFSIZE", + /* 190 */ "STREAM", + /* 191 */ "INTO", + /* 192 */ "TRIGGER", + /* 193 */ "AT_ONCE", + /* 194 */ "WINDOW_CLOSE", + /* 195 */ "IGNORE", + /* 196 */ "EXPIRED", + /* 197 */ "FILL_HISTORY", + /* 198 */ "UPDATE", + /* 199 */ "SUBTABLE", + /* 200 */ "KILL", + /* 201 */ "CONNECTION", + /* 202 */ "TRANSACTION", + /* 203 */ "BALANCE", + /* 204 */ "VGROUP", + /* 205 */ "MERGE", + /* 206 */ "REDISTRIBUTE", + /* 207 */ "SPLIT", + /* 208 */ "DELETE", + /* 209 */ "INSERT", + /* 210 */ "NULL", + /* 211 */ "NK_QUESTION", + /* 212 */ "NK_ARROW", + /* 213 */ "ROWTS", + /* 214 */ "QSTART", + /* 215 */ "QEND", + /* 216 */ "QDURATION", + /* 217 */ "WSTART", + /* 218 */ "WEND", + /* 219 */ "WDURATION", + /* 220 */ "IROWTS", + /* 221 */ "ISFILLED", + /* 222 */ "CAST", + /* 223 */ "NOW", + /* 224 */ "TODAY", + /* 225 */ "TIMEZONE", + /* 226 */ "CLIENT_VERSION", + /* 227 */ "SERVER_VERSION", + /* 228 */ "SERVER_STATUS", + /* 229 */ "CURRENT_USER", + /* 230 */ "CASE", + /* 231 */ "END", + /* 232 */ "WHEN", + /* 233 */ "THEN", + /* 234 */ "ELSE", + /* 235 */ "BETWEEN", + /* 236 */ "IS", + /* 237 */ "NK_LT", + /* 238 */ "NK_GT", + /* 239 */ "NK_LE", + /* 240 */ "NK_GE", + /* 241 */ "NK_NE", + /* 242 */ "MATCH", + /* 243 */ "NMATCH", + /* 244 */ "CONTAINS", + /* 245 */ "IN", + /* 246 */ "JOIN", + /* 247 */ "INNER", + /* 248 */ "SELECT", + /* 249 */ "DISTINCT", + /* 250 */ "WHERE", + /* 251 */ "PARTITION", + /* 252 */ "BY", + /* 253 */ "SESSION", + /* 254 */ "STATE_WINDOW", + /* 255 */ "EVENT_WINDOW", + /* 256 */ "START", + /* 257 */ "SLIDING", + /* 258 */ "FILL", + /* 259 */ "VALUE", + /* 260 */ "VALUE_F", + /* 261 */ "NONE", + /* 262 */ "PREV", + /* 263 */ "NULL_F", + /* 264 */ "LINEAR", + /* 265 */ "NEXT", + /* 266 */ "HAVING", + /* 267 */ "RANGE", + /* 268 */ "EVERY", + /* 269 */ "ORDER", + /* 270 */ "SLIMIT", + /* 271 */ "SOFFSET", + /* 272 */ "LIMIT", + /* 273 */ "OFFSET", + /* 274 */ "ASC", + /* 275 */ "NULLS", + /* 276 */ "ABORT", + /* 277 */ "AFTER", + /* 278 */ "ATTACH", + /* 279 */ "BEFORE", + /* 280 */ "BEGIN", + /* 281 */ "BITAND", + /* 282 */ "BITNOT", + /* 283 */ "BITOR", + /* 284 */ "BLOCKS", + /* 285 */ "CHANGE", + /* 286 */ "COMMA", + /* 287 */ "CONCAT", + /* 288 */ "CONFLICT", + /* 289 */ "COPY", + /* 290 */ "DEFERRED", + /* 291 */ "DELIMITERS", + /* 292 */ "DETACH", + /* 293 */ "DIVIDE", + /* 294 */ "DOT", + /* 295 */ "EACH", + /* 296 */ "FAIL", + /* 297 */ "FILE", + /* 298 */ "FOR", + /* 299 */ "GLOB", + /* 300 */ "ID", + /* 301 */ "IMMEDIATE", + /* 302 */ "IMPORT", + /* 303 */ "INITIALLY", + /* 304 */ "INSTEAD", + /* 305 */ "ISNULL", + /* 306 */ "KEY", + /* 307 */ "MODULES", + /* 308 */ "NK_BITNOT", + /* 309 */ "NK_SEMI", + /* 310 */ "NOTNULL", + /* 311 */ "OF", + /* 312 */ "PLUS", + /* 313 */ "PRIVILEGE", + /* 314 */ "RAISE", + /* 315 */ "REPLACE", + /* 316 */ "RESTRICT", + /* 317 */ "ROW", + /* 318 */ "SEMI", + /* 319 */ "STAR", + /* 320 */ "STATEMENT", + /* 321 */ "STRICT", + /* 322 */ "STRING", + /* 323 */ "TIMES", + /* 324 */ "VALUES", + /* 325 */ "VARIABLE", + /* 326 */ "VIEW", + /* 327 */ "WAL", + /* 328 */ "cmd", + /* 329 */ "account_options", + /* 330 */ "alter_account_options", + /* 331 */ "literal", + /* 332 */ "alter_account_option", + /* 333 */ "user_name", + /* 334 */ "sysinfo_opt", + /* 335 */ "privileges", + /* 336 */ "priv_level", + /* 337 */ "priv_type_list", + /* 338 */ "priv_type", + /* 339 */ "db_name", + /* 340 */ "topic_name", + /* 341 */ "dnode_endpoint", + /* 342 */ "force_opt", + /* 343 */ "not_exists_opt", + /* 344 */ "db_options", + /* 345 */ "exists_opt", + /* 346 */ "alter_db_options", + /* 347 */ "speed_opt", + /* 348 */ "integer_list", + /* 349 */ "variable_list", + /* 350 */ "retention_list", + /* 351 */ "alter_db_option", + /* 352 */ "retention", + /* 353 */ "full_table_name", + /* 354 */ "column_def_list", + /* 355 */ "tags_def_opt", + /* 356 */ "table_options", + /* 357 */ "multi_create_clause", + /* 358 */ "tags_def", + /* 359 */ "multi_drop_clause", + /* 360 */ "alter_table_clause", + /* 361 */ "alter_table_options", + /* 362 */ "column_name", + /* 363 */ "type_name", + /* 364 */ "signed_literal", + /* 365 */ "create_subtable_clause", + /* 366 */ "specific_cols_opt", + /* 367 */ "expression_list", + /* 368 */ "drop_table_clause", + /* 369 */ "col_name_list", + /* 370 */ "table_name", + /* 371 */ "column_def", + /* 372 */ "duration_list", + /* 373 */ "rollup_func_list", + /* 374 */ "alter_table_option", + /* 375 */ "duration_literal", + /* 376 */ "rollup_func_name", + /* 377 */ "function_name", + /* 378 */ "col_name", + /* 379 */ "db_name_cond_opt", + /* 380 */ "like_pattern_opt", + /* 381 */ "table_name_cond", + /* 382 */ "from_db_opt", + /* 383 */ "tag_list_opt", + /* 384 */ "tag_item", + /* 385 */ "column_alias", + /* 386 */ "full_index_name", + /* 387 */ "index_options", + /* 388 */ "index_name", + /* 389 */ "func_list", + /* 390 */ "sliding_opt", + /* 391 */ "sma_stream_opt", + /* 392 */ "func", + /* 393 */ "sma_func_name", + /* 394 */ "query_or_subquery", + /* 395 */ "cgroup_name", + /* 396 */ "analyze_opt", + /* 397 */ "explain_options", + /* 398 */ "insert_query", + /* 399 */ "agg_func_opt", + /* 400 */ "bufsize_opt", + /* 401 */ "stream_name", + /* 402 */ "stream_options", + /* 403 */ "col_list_opt", + /* 404 */ "tag_def_or_ref_opt", + /* 405 */ "subtable_opt", + /* 406 */ "expression", + /* 407 */ "dnode_list", + /* 408 */ "where_clause_opt", + /* 409 */ "signed", + /* 410 */ "literal_func", + /* 411 */ "literal_list", + /* 412 */ "table_alias", + /* 413 */ "expr_or_subquery", + /* 414 */ "pseudo_column", + /* 415 */ "column_reference", + /* 416 */ "function_expression", + /* 417 */ "case_when_expression", + /* 418 */ "star_func", + /* 419 */ "star_func_para_list", + /* 420 */ "noarg_func", + /* 421 */ "other_para_list", + /* 422 */ "star_func_para", + /* 423 */ "when_then_list", + /* 424 */ "case_when_else_opt", + /* 425 */ "common_expression", + /* 426 */ "when_then_expr", + /* 427 */ "predicate", + /* 428 */ "compare_op", + /* 429 */ "in_op", + /* 430 */ "in_predicate_value", + /* 431 */ "boolean_value_expression", + /* 432 */ "boolean_primary", + /* 433 */ "from_clause_opt", + /* 434 */ "table_reference_list", + /* 435 */ "table_reference", + /* 436 */ "table_primary", + /* 437 */ "joined_table", + /* 438 */ "alias_opt", + /* 439 */ "subquery", + /* 440 */ "parenthesized_joined_table", + /* 441 */ "join_type", + /* 442 */ "search_condition", + /* 443 */ "query_specification", + /* 444 */ "set_quantifier_opt", + /* 445 */ "select_list", + /* 446 */ "partition_by_clause_opt", + /* 447 */ "range_opt", + /* 448 */ "every_opt", + /* 449 */ "fill_opt", + /* 450 */ "twindow_clause_opt", + /* 451 */ "group_by_clause_opt", + /* 452 */ "having_clause_opt", + /* 453 */ "select_item", + /* 454 */ "partition_list", + /* 455 */ "partition_item", + /* 456 */ "fill_mode", + /* 457 */ "group_by_list", + /* 458 */ "query_expression", + /* 459 */ "query_simple", + /* 460 */ "order_by_clause_opt", + /* 461 */ "slimit_clause_opt", + /* 462 */ "limit_clause_opt", + /* 463 */ "union_query_expression", + /* 464 */ "query_simple_or_subquery", + /* 465 */ "sort_specification_list", + /* 466 */ "sort_specification", + /* 467 */ "ordering_specification_opt", + /* 468 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1924,479 +1945,497 @@ static const char *const yyRuleName[] = { /* 69 */ "cmd ::= ALTER DATABASE db_name alter_db_options", /* 70 */ "cmd ::= FLUSH DATABASE db_name", /* 71 */ "cmd ::= TRIM DATABASE db_name speed_opt", - /* 72 */ "not_exists_opt ::= IF NOT EXISTS", - /* 73 */ "not_exists_opt ::=", - /* 74 */ "exists_opt ::= IF EXISTS", - /* 75 */ "exists_opt ::=", - /* 76 */ "db_options ::=", - /* 77 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 78 */ "db_options ::= db_options CACHEMODEL NK_STRING", - /* 79 */ "db_options ::= db_options CACHESIZE NK_INTEGER", - /* 80 */ "db_options ::= db_options COMP NK_INTEGER", - /* 81 */ "db_options ::= db_options DURATION NK_INTEGER", - /* 82 */ "db_options ::= db_options DURATION NK_VARIABLE", - /* 83 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 84 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 85 */ "db_options ::= db_options KEEP integer_list", - /* 86 */ "db_options ::= db_options KEEP variable_list", - /* 87 */ "db_options ::= db_options PAGES NK_INTEGER", - /* 88 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 89 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", - /* 90 */ "db_options ::= db_options PRECISION NK_STRING", - /* 91 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 92 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 93 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 94 */ "db_options ::= db_options RETENTIONS retention_list", - /* 95 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 96 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 97 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 98 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 100 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 102 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 103 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 104 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 105 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 106 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", - /* 107 */ "alter_db_options ::= alter_db_option", - /* 108 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 109 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 110 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 111 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 112 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 113 */ "alter_db_option ::= KEEP integer_list", - /* 114 */ "alter_db_option ::= KEEP variable_list", - /* 115 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 116 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 117 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 118 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", - /* 119 */ "integer_list ::= NK_INTEGER", - /* 120 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 121 */ "variable_list ::= NK_VARIABLE", - /* 122 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 123 */ "retention_list ::= retention", - /* 124 */ "retention_list ::= retention_list NK_COMMA retention", - /* 125 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 126 */ "speed_opt ::=", - /* 127 */ "speed_opt ::= MAX_SPEED NK_INTEGER", - /* 128 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 129 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 130 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 131 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 132 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 133 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 134 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 135 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 136 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 137 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 138 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 139 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 140 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 141 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 142 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 143 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 144 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 145 */ "multi_create_clause ::= create_subtable_clause", - /* 146 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 147 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 148 */ "multi_drop_clause ::= drop_table_clause", - /* 149 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 150 */ "drop_table_clause ::= exists_opt full_table_name", - /* 151 */ "specific_cols_opt ::=", - /* 152 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 153 */ "full_table_name ::= table_name", - /* 154 */ "full_table_name ::= db_name NK_DOT table_name", - /* 155 */ "column_def_list ::= column_def", - /* 156 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 157 */ "column_def ::= column_name type_name", - /* 158 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 159 */ "type_name ::= BOOL", - /* 160 */ "type_name ::= TINYINT", - /* 161 */ "type_name ::= SMALLINT", - /* 162 */ "type_name ::= INT", - /* 163 */ "type_name ::= INTEGER", - /* 164 */ "type_name ::= BIGINT", - /* 165 */ "type_name ::= FLOAT", - /* 166 */ "type_name ::= DOUBLE", - /* 167 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 168 */ "type_name ::= TIMESTAMP", - /* 169 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 170 */ "type_name ::= TINYINT UNSIGNED", - /* 171 */ "type_name ::= SMALLINT UNSIGNED", - /* 172 */ "type_name ::= INT UNSIGNED", - /* 173 */ "type_name ::= BIGINT UNSIGNED", - /* 174 */ "type_name ::= JSON", - /* 175 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 176 */ "type_name ::= MEDIUMBLOB", - /* 177 */ "type_name ::= BLOB", - /* 178 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 179 */ "type_name ::= DECIMAL", - /* 180 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 181 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 182 */ "tags_def_opt ::=", - /* 183 */ "tags_def_opt ::= tags_def", - /* 184 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 185 */ "table_options ::=", - /* 186 */ "table_options ::= table_options COMMENT NK_STRING", - /* 187 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 188 */ "table_options ::= table_options WATERMARK duration_list", - /* 189 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 190 */ "table_options ::= table_options TTL NK_INTEGER", - /* 191 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 192 */ "table_options ::= table_options DELETE_MARK duration_list", - /* 193 */ "alter_table_options ::= alter_table_option", - /* 194 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 195 */ "alter_table_option ::= COMMENT NK_STRING", - /* 196 */ "alter_table_option ::= TTL NK_INTEGER", - /* 197 */ "duration_list ::= duration_literal", - /* 198 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 199 */ "rollup_func_list ::= rollup_func_name", - /* 200 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 201 */ "rollup_func_name ::= function_name", - /* 202 */ "rollup_func_name ::= FIRST", - /* 203 */ "rollup_func_name ::= LAST", - /* 204 */ "col_name_list ::= col_name", - /* 205 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 206 */ "col_name ::= column_name", - /* 207 */ "cmd ::= SHOW DNODES", - /* 208 */ "cmd ::= SHOW USERS", - /* 209 */ "cmd ::= SHOW USER PRIVILEGES", - /* 210 */ "cmd ::= SHOW DATABASES", - /* 211 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 212 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 213 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 214 */ "cmd ::= SHOW MNODES", - /* 215 */ "cmd ::= SHOW QNODES", - /* 216 */ "cmd ::= SHOW FUNCTIONS", - /* 217 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 218 */ "cmd ::= SHOW STREAMS", - /* 219 */ "cmd ::= SHOW ACCOUNTS", - /* 220 */ "cmd ::= SHOW APPS", - /* 221 */ "cmd ::= SHOW CONNECTIONS", - /* 222 */ "cmd ::= SHOW LICENCES", - /* 223 */ "cmd ::= SHOW GRANTS", - /* 224 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 225 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 226 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 227 */ "cmd ::= SHOW QUERIES", - /* 228 */ "cmd ::= SHOW SCORES", - /* 229 */ "cmd ::= SHOW TOPICS", - /* 230 */ "cmd ::= SHOW VARIABLES", - /* 231 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 232 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 233 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 234 */ "cmd ::= SHOW BNODES", - /* 235 */ "cmd ::= SHOW SNODES", - /* 236 */ "cmd ::= SHOW CLUSTER", - /* 237 */ "cmd ::= SHOW TRANSACTIONS", - /* 238 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 239 */ "cmd ::= SHOW CONSUMERS", - /* 240 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 241 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 242 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 243 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 244 */ "cmd ::= SHOW VNODES NK_STRING", - /* 245 */ "db_name_cond_opt ::=", - /* 246 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 247 */ "like_pattern_opt ::=", - /* 248 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 249 */ "table_name_cond ::= table_name", - /* 250 */ "from_db_opt ::=", - /* 251 */ "from_db_opt ::= FROM db_name", - /* 252 */ "tag_list_opt ::=", - /* 253 */ "tag_list_opt ::= tag_item", - /* 254 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 255 */ "tag_item ::= TBNAME", - /* 256 */ "tag_item ::= QTAGS", - /* 257 */ "tag_item ::= column_name", - /* 258 */ "tag_item ::= column_name column_alias", - /* 259 */ "tag_item ::= column_name AS column_alias", - /* 260 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 261 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 262 */ "full_index_name ::= index_name", - /* 263 */ "full_index_name ::= db_name NK_DOT index_name", - /* 264 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 265 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 266 */ "func_list ::= func", - /* 267 */ "func_list ::= func_list NK_COMMA func", - /* 268 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 269 */ "sma_stream_opt ::=", - /* 270 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 271 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 272 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 273 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 274 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 275 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 276 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 277 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 278 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 279 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 280 */ "cmd ::= DESC full_table_name", - /* 281 */ "cmd ::= DESCRIBE full_table_name", - /* 282 */ "cmd ::= RESET QUERY CACHE", - /* 283 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 284 */ "analyze_opt ::=", - /* 285 */ "analyze_opt ::= ANALYZE", - /* 286 */ "explain_options ::=", - /* 287 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 288 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 289 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 290 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 291 */ "agg_func_opt ::=", - /* 292 */ "agg_func_opt ::= AGGREGATE", - /* 293 */ "bufsize_opt ::=", - /* 294 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 295 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery", - /* 296 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 297 */ "stream_options ::=", - /* 298 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 299 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 300 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 301 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 302 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 303 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 304 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 305 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 306 */ "subtable_opt ::=", - /* 307 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 308 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 309 */ "cmd ::= KILL QUERY NK_STRING", - /* 310 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 311 */ "cmd ::= BALANCE VGROUP", - /* 312 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 313 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 314 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 315 */ "dnode_list ::= DNODE NK_INTEGER", - /* 316 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 317 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 318 */ "cmd ::= query_or_subquery", - /* 319 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 320 */ "cmd ::= INSERT INTO full_table_name query_or_subquery", - /* 321 */ "literal ::= NK_INTEGER", - /* 322 */ "literal ::= NK_FLOAT", - /* 323 */ "literal ::= NK_STRING", - /* 324 */ "literal ::= NK_BOOL", - /* 325 */ "literal ::= TIMESTAMP NK_STRING", - /* 326 */ "literal ::= duration_literal", - /* 327 */ "literal ::= NULL", - /* 328 */ "literal ::= NK_QUESTION", - /* 329 */ "duration_literal ::= NK_VARIABLE", - /* 330 */ "signed ::= NK_INTEGER", - /* 331 */ "signed ::= NK_PLUS NK_INTEGER", - /* 332 */ "signed ::= NK_MINUS NK_INTEGER", - /* 333 */ "signed ::= NK_FLOAT", - /* 334 */ "signed ::= NK_PLUS NK_FLOAT", - /* 335 */ "signed ::= NK_MINUS NK_FLOAT", - /* 336 */ "signed_literal ::= signed", - /* 337 */ "signed_literal ::= NK_STRING", - /* 338 */ "signed_literal ::= NK_BOOL", - /* 339 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 340 */ "signed_literal ::= duration_literal", - /* 341 */ "signed_literal ::= NULL", - /* 342 */ "signed_literal ::= literal_func", - /* 343 */ "signed_literal ::= NK_QUESTION", - /* 344 */ "literal_list ::= signed_literal", - /* 345 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 346 */ "db_name ::= NK_ID", - /* 347 */ "table_name ::= NK_ID", - /* 348 */ "column_name ::= NK_ID", - /* 349 */ "function_name ::= NK_ID", - /* 350 */ "table_alias ::= NK_ID", - /* 351 */ "column_alias ::= NK_ID", - /* 352 */ "user_name ::= NK_ID", - /* 353 */ "topic_name ::= NK_ID", - /* 354 */ "stream_name ::= NK_ID", - /* 355 */ "cgroup_name ::= NK_ID", - /* 356 */ "index_name ::= NK_ID", - /* 357 */ "expr_or_subquery ::= expression", - /* 358 */ "expression ::= literal", - /* 359 */ "expression ::= pseudo_column", - /* 360 */ "expression ::= column_reference", - /* 361 */ "expression ::= function_expression", - /* 362 */ "expression ::= case_when_expression", - /* 363 */ "expression ::= NK_LP expression NK_RP", - /* 364 */ "expression ::= NK_PLUS expr_or_subquery", - /* 365 */ "expression ::= NK_MINUS expr_or_subquery", - /* 366 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 367 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 368 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 369 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 370 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 371 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 372 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 373 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 374 */ "expression_list ::= expr_or_subquery", - /* 375 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 376 */ "column_reference ::= column_name", - /* 377 */ "column_reference ::= table_name NK_DOT column_name", - /* 378 */ "pseudo_column ::= ROWTS", - /* 379 */ "pseudo_column ::= TBNAME", - /* 380 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 381 */ "pseudo_column ::= QSTART", - /* 382 */ "pseudo_column ::= QEND", - /* 383 */ "pseudo_column ::= QDURATION", - /* 384 */ "pseudo_column ::= WSTART", - /* 385 */ "pseudo_column ::= WEND", - /* 386 */ "pseudo_column ::= WDURATION", - /* 387 */ "pseudo_column ::= IROWTS", - /* 388 */ "pseudo_column ::= QTAGS", - /* 389 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 390 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 391 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 392 */ "function_expression ::= literal_func", - /* 393 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 394 */ "literal_func ::= NOW", - /* 395 */ "noarg_func ::= NOW", - /* 396 */ "noarg_func ::= TODAY", - /* 397 */ "noarg_func ::= TIMEZONE", - /* 398 */ "noarg_func ::= DATABASE", - /* 399 */ "noarg_func ::= CLIENT_VERSION", - /* 400 */ "noarg_func ::= SERVER_VERSION", - /* 401 */ "noarg_func ::= SERVER_STATUS", - /* 402 */ "noarg_func ::= CURRENT_USER", - /* 403 */ "noarg_func ::= USER", - /* 404 */ "star_func ::= COUNT", - /* 405 */ "star_func ::= FIRST", - /* 406 */ "star_func ::= LAST", - /* 407 */ "star_func ::= LAST_ROW", - /* 408 */ "star_func_para_list ::= NK_STAR", - /* 409 */ "star_func_para_list ::= other_para_list", - /* 410 */ "other_para_list ::= star_func_para", - /* 411 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 412 */ "star_func_para ::= expr_or_subquery", - /* 413 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 414 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 415 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 416 */ "when_then_list ::= when_then_expr", - /* 417 */ "when_then_list ::= when_then_list when_then_expr", - /* 418 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 419 */ "case_when_else_opt ::=", - /* 420 */ "case_when_else_opt ::= ELSE common_expression", - /* 421 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 422 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 423 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 424 */ "predicate ::= expr_or_subquery IS NULL", - /* 425 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 426 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 427 */ "compare_op ::= NK_LT", - /* 428 */ "compare_op ::= NK_GT", - /* 429 */ "compare_op ::= NK_LE", - /* 430 */ "compare_op ::= NK_GE", - /* 431 */ "compare_op ::= NK_NE", - /* 432 */ "compare_op ::= NK_EQ", - /* 433 */ "compare_op ::= LIKE", - /* 434 */ "compare_op ::= NOT LIKE", - /* 435 */ "compare_op ::= MATCH", - /* 436 */ "compare_op ::= NMATCH", - /* 437 */ "compare_op ::= CONTAINS", - /* 438 */ "in_op ::= IN", - /* 439 */ "in_op ::= NOT IN", - /* 440 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 441 */ "boolean_value_expression ::= boolean_primary", - /* 442 */ "boolean_value_expression ::= NOT boolean_primary", - /* 443 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 444 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 445 */ "boolean_primary ::= predicate", - /* 446 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 447 */ "common_expression ::= expr_or_subquery", - /* 448 */ "common_expression ::= boolean_value_expression", - /* 449 */ "from_clause_opt ::=", - /* 450 */ "from_clause_opt ::= FROM table_reference_list", - /* 451 */ "table_reference_list ::= table_reference", - /* 452 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 453 */ "table_reference ::= table_primary", - /* 454 */ "table_reference ::= joined_table", - /* 455 */ "table_primary ::= table_name alias_opt", - /* 456 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 457 */ "table_primary ::= subquery alias_opt", - /* 458 */ "table_primary ::= parenthesized_joined_table", - /* 459 */ "alias_opt ::=", - /* 460 */ "alias_opt ::= table_alias", - /* 461 */ "alias_opt ::= AS table_alias", - /* 462 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 463 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 464 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 465 */ "join_type ::=", - /* 466 */ "join_type ::= INNER", - /* 467 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 468 */ "set_quantifier_opt ::=", - /* 469 */ "set_quantifier_opt ::= DISTINCT", - /* 470 */ "set_quantifier_opt ::= ALL", - /* 471 */ "select_list ::= select_item", - /* 472 */ "select_list ::= select_list NK_COMMA select_item", - /* 473 */ "select_item ::= NK_STAR", - /* 474 */ "select_item ::= common_expression", - /* 475 */ "select_item ::= common_expression column_alias", - /* 476 */ "select_item ::= common_expression AS column_alias", - /* 477 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 478 */ "where_clause_opt ::=", - /* 479 */ "where_clause_opt ::= WHERE search_condition", - /* 480 */ "partition_by_clause_opt ::=", - /* 481 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 482 */ "partition_list ::= partition_item", - /* 483 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 484 */ "partition_item ::= expr_or_subquery", - /* 485 */ "partition_item ::= expr_or_subquery column_alias", - /* 486 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 487 */ "twindow_clause_opt ::=", - /* 488 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 489 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 490 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 491 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 492 */ "sliding_opt ::=", - /* 493 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 494 */ "fill_opt ::=", - /* 495 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 496 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 497 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", - /* 498 */ "fill_mode ::= NONE", - /* 499 */ "fill_mode ::= PREV", - /* 500 */ "fill_mode ::= NULL", - /* 501 */ "fill_mode ::= NULL_F", - /* 502 */ "fill_mode ::= LINEAR", - /* 503 */ "fill_mode ::= NEXT", - /* 504 */ "group_by_clause_opt ::=", - /* 505 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 506 */ "group_by_list ::= expr_or_subquery", - /* 507 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 508 */ "having_clause_opt ::=", - /* 509 */ "having_clause_opt ::= HAVING search_condition", - /* 510 */ "range_opt ::=", - /* 511 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 512 */ "every_opt ::=", - /* 513 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 514 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 515 */ "query_simple ::= query_specification", - /* 516 */ "query_simple ::= union_query_expression", - /* 517 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 518 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 519 */ "query_simple_or_subquery ::= query_simple", - /* 520 */ "query_simple_or_subquery ::= subquery", - /* 521 */ "query_or_subquery ::= query_expression", - /* 522 */ "query_or_subquery ::= subquery", - /* 523 */ "order_by_clause_opt ::=", - /* 524 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 525 */ "slimit_clause_opt ::=", - /* 526 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 527 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 528 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 529 */ "limit_clause_opt ::=", - /* 530 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 531 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 532 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 533 */ "subquery ::= NK_LP query_expression NK_RP", - /* 534 */ "subquery ::= NK_LP subquery NK_RP", - /* 535 */ "search_condition ::= common_expression", - /* 536 */ "sort_specification_list ::= sort_specification", - /* 537 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 538 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 539 */ "ordering_specification_opt ::=", - /* 540 */ "ordering_specification_opt ::= ASC", - /* 541 */ "ordering_specification_opt ::= DESC", - /* 542 */ "null_ordering_opt ::=", - /* 543 */ "null_ordering_opt ::= NULLS FIRST", - /* 544 */ "null_ordering_opt ::= NULLS LAST", + /* 72 */ "cmd ::= COMPACT DATABASE db_name", + /* 73 */ "not_exists_opt ::= IF NOT EXISTS", + /* 74 */ "not_exists_opt ::=", + /* 75 */ "exists_opt ::= IF EXISTS", + /* 76 */ "exists_opt ::=", + /* 77 */ "db_options ::=", + /* 78 */ "db_options ::= db_options BUFFER NK_INTEGER", + /* 79 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 80 */ "db_options ::= db_options CACHESIZE NK_INTEGER", + /* 81 */ "db_options ::= db_options COMP NK_INTEGER", + /* 82 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 83 */ "db_options ::= db_options DURATION NK_VARIABLE", + /* 84 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 85 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 86 */ "db_options ::= db_options KEEP integer_list", + /* 87 */ "db_options ::= db_options KEEP variable_list", + /* 88 */ "db_options ::= db_options PAGES NK_INTEGER", + /* 89 */ "db_options ::= db_options PAGESIZE NK_INTEGER", + /* 90 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", + /* 91 */ "db_options ::= db_options PRECISION NK_STRING", + /* 92 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 93 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 94 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 95 */ "db_options ::= db_options RETENTIONS retention_list", + /* 96 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 97 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 98 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 100 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 102 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 103 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 104 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 105 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 106 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", + /* 107 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 108 */ "alter_db_options ::= alter_db_option", + /* 109 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 110 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 111 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 112 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 113 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 114 */ "alter_db_option ::= KEEP integer_list", + /* 115 */ "alter_db_option ::= KEEP variable_list", + /* 116 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 117 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 118 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 119 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 120 */ "integer_list ::= NK_INTEGER", + /* 121 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 122 */ "variable_list ::= NK_VARIABLE", + /* 123 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 124 */ "retention_list ::= retention", + /* 125 */ "retention_list ::= retention_list NK_COMMA retention", + /* 126 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 127 */ "speed_opt ::=", + /* 128 */ "speed_opt ::= MAX_SPEED NK_INTEGER", + /* 129 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 130 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 131 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 132 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 133 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 134 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 135 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 136 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 137 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 138 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 139 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 140 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 141 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 142 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 143 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 144 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 145 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 146 */ "multi_create_clause ::= create_subtable_clause", + /* 147 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 148 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 149 */ "multi_drop_clause ::= drop_table_clause", + /* 150 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", + /* 151 */ "drop_table_clause ::= exists_opt full_table_name", + /* 152 */ "specific_cols_opt ::=", + /* 153 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 154 */ "full_table_name ::= table_name", + /* 155 */ "full_table_name ::= db_name NK_DOT table_name", + /* 156 */ "column_def_list ::= column_def", + /* 157 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 158 */ "column_def ::= column_name type_name", + /* 159 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 160 */ "type_name ::= BOOL", + /* 161 */ "type_name ::= TINYINT", + /* 162 */ "type_name ::= SMALLINT", + /* 163 */ "type_name ::= INT", + /* 164 */ "type_name ::= INTEGER", + /* 165 */ "type_name ::= BIGINT", + /* 166 */ "type_name ::= FLOAT", + /* 167 */ "type_name ::= DOUBLE", + /* 168 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 169 */ "type_name ::= TIMESTAMP", + /* 170 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 171 */ "type_name ::= TINYINT UNSIGNED", + /* 172 */ "type_name ::= SMALLINT UNSIGNED", + /* 173 */ "type_name ::= INT UNSIGNED", + /* 174 */ "type_name ::= BIGINT UNSIGNED", + /* 175 */ "type_name ::= JSON", + /* 176 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 177 */ "type_name ::= MEDIUMBLOB", + /* 178 */ "type_name ::= BLOB", + /* 179 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 180 */ "type_name ::= DECIMAL", + /* 181 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 182 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 183 */ "tags_def_opt ::=", + /* 184 */ "tags_def_opt ::= tags_def", + /* 185 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 186 */ "table_options ::=", + /* 187 */ "table_options ::= table_options COMMENT NK_STRING", + /* 188 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 189 */ "table_options ::= table_options WATERMARK duration_list", + /* 190 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 191 */ "table_options ::= table_options TTL NK_INTEGER", + /* 192 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 193 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 194 */ "alter_table_options ::= alter_table_option", + /* 195 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 196 */ "alter_table_option ::= COMMENT NK_STRING", + /* 197 */ "alter_table_option ::= TTL NK_INTEGER", + /* 198 */ "duration_list ::= duration_literal", + /* 199 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 200 */ "rollup_func_list ::= rollup_func_name", + /* 201 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 202 */ "rollup_func_name ::= function_name", + /* 203 */ "rollup_func_name ::= FIRST", + /* 204 */ "rollup_func_name ::= LAST", + /* 205 */ "col_name_list ::= col_name", + /* 206 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 207 */ "col_name ::= column_name", + /* 208 */ "cmd ::= SHOW DNODES", + /* 209 */ "cmd ::= SHOW USERS", + /* 210 */ "cmd ::= SHOW USER PRIVILEGES", + /* 211 */ "cmd ::= SHOW DATABASES", + /* 212 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 213 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 214 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 215 */ "cmd ::= SHOW MNODES", + /* 216 */ "cmd ::= SHOW QNODES", + /* 217 */ "cmd ::= SHOW FUNCTIONS", + /* 218 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 219 */ "cmd ::= SHOW STREAMS", + /* 220 */ "cmd ::= SHOW ACCOUNTS", + /* 221 */ "cmd ::= SHOW APPS", + /* 222 */ "cmd ::= SHOW CONNECTIONS", + /* 223 */ "cmd ::= SHOW LICENCES", + /* 224 */ "cmd ::= SHOW GRANTS", + /* 225 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 226 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 227 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 228 */ "cmd ::= SHOW QUERIES", + /* 229 */ "cmd ::= SHOW SCORES", + /* 230 */ "cmd ::= SHOW TOPICS", + /* 231 */ "cmd ::= SHOW VARIABLES", + /* 232 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 233 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 234 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 235 */ "cmd ::= SHOW BNODES", + /* 236 */ "cmd ::= SHOW SNODES", + /* 237 */ "cmd ::= SHOW CLUSTER", + /* 238 */ "cmd ::= SHOW TRANSACTIONS", + /* 239 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 240 */ "cmd ::= SHOW CONSUMERS", + /* 241 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 242 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 243 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 244 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 245 */ "cmd ::= SHOW VNODES NK_STRING", + /* 246 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 247 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 248 */ "db_name_cond_opt ::=", + /* 249 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 250 */ "like_pattern_opt ::=", + /* 251 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 252 */ "table_name_cond ::= table_name", + /* 253 */ "from_db_opt ::=", + /* 254 */ "from_db_opt ::= FROM db_name", + /* 255 */ "tag_list_opt ::=", + /* 256 */ "tag_list_opt ::= tag_item", + /* 257 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 258 */ "tag_item ::= TBNAME", + /* 259 */ "tag_item ::= QTAGS", + /* 260 */ "tag_item ::= column_name", + /* 261 */ "tag_item ::= column_name column_alias", + /* 262 */ "tag_item ::= column_name AS column_alias", + /* 263 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 264 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 265 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 266 */ "full_index_name ::= index_name", + /* 267 */ "full_index_name ::= db_name NK_DOT index_name", + /* 268 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 269 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 270 */ "func_list ::= func", + /* 271 */ "func_list ::= func_list NK_COMMA func", + /* 272 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 273 */ "sma_func_name ::= function_name", + /* 274 */ "sma_func_name ::= COUNT", + /* 275 */ "sma_func_name ::= FIRST", + /* 276 */ "sma_func_name ::= LAST", + /* 277 */ "sma_func_name ::= LAST_ROW", + /* 278 */ "sma_stream_opt ::=", + /* 279 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 280 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 281 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 282 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 283 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 284 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 285 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 286 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 287 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 288 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 289 */ "cmd ::= DESC full_table_name", + /* 290 */ "cmd ::= DESCRIBE full_table_name", + /* 291 */ "cmd ::= RESET QUERY CACHE", + /* 292 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 293 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 294 */ "analyze_opt ::=", + /* 295 */ "analyze_opt ::= ANALYZE", + /* 296 */ "explain_options ::=", + /* 297 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 298 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 299 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 300 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 301 */ "agg_func_opt ::=", + /* 302 */ "agg_func_opt ::= AGGREGATE", + /* 303 */ "bufsize_opt ::=", + /* 304 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 305 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 306 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 307 */ "col_list_opt ::=", + /* 308 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 309 */ "tag_def_or_ref_opt ::=", + /* 310 */ "tag_def_or_ref_opt ::= tags_def", + /* 311 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 312 */ "stream_options ::=", + /* 313 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 314 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 315 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 316 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 317 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 318 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 319 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 320 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 321 */ "subtable_opt ::=", + /* 322 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 323 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 324 */ "cmd ::= KILL QUERY NK_STRING", + /* 325 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 326 */ "cmd ::= BALANCE VGROUP", + /* 327 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 328 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 329 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 330 */ "dnode_list ::= DNODE NK_INTEGER", + /* 331 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 332 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 333 */ "cmd ::= query_or_subquery", + /* 334 */ "cmd ::= insert_query", + /* 335 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 336 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 337 */ "literal ::= NK_INTEGER", + /* 338 */ "literal ::= NK_FLOAT", + /* 339 */ "literal ::= NK_STRING", + /* 340 */ "literal ::= NK_BOOL", + /* 341 */ "literal ::= TIMESTAMP NK_STRING", + /* 342 */ "literal ::= duration_literal", + /* 343 */ "literal ::= NULL", + /* 344 */ "literal ::= NK_QUESTION", + /* 345 */ "duration_literal ::= NK_VARIABLE", + /* 346 */ "signed ::= NK_INTEGER", + /* 347 */ "signed ::= NK_PLUS NK_INTEGER", + /* 348 */ "signed ::= NK_MINUS NK_INTEGER", + /* 349 */ "signed ::= NK_FLOAT", + /* 350 */ "signed ::= NK_PLUS NK_FLOAT", + /* 351 */ "signed ::= NK_MINUS NK_FLOAT", + /* 352 */ "signed_literal ::= signed", + /* 353 */ "signed_literal ::= NK_STRING", + /* 354 */ "signed_literal ::= NK_BOOL", + /* 355 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 356 */ "signed_literal ::= duration_literal", + /* 357 */ "signed_literal ::= NULL", + /* 358 */ "signed_literal ::= literal_func", + /* 359 */ "signed_literal ::= NK_QUESTION", + /* 360 */ "literal_list ::= signed_literal", + /* 361 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 362 */ "db_name ::= NK_ID", + /* 363 */ "table_name ::= NK_ID", + /* 364 */ "column_name ::= NK_ID", + /* 365 */ "function_name ::= NK_ID", + /* 366 */ "table_alias ::= NK_ID", + /* 367 */ "column_alias ::= NK_ID", + /* 368 */ "user_name ::= NK_ID", + /* 369 */ "topic_name ::= NK_ID", + /* 370 */ "stream_name ::= NK_ID", + /* 371 */ "cgroup_name ::= NK_ID", + /* 372 */ "index_name ::= NK_ID", + /* 373 */ "expr_or_subquery ::= expression", + /* 374 */ "expression ::= literal", + /* 375 */ "expression ::= pseudo_column", + /* 376 */ "expression ::= column_reference", + /* 377 */ "expression ::= function_expression", + /* 378 */ "expression ::= case_when_expression", + /* 379 */ "expression ::= NK_LP expression NK_RP", + /* 380 */ "expression ::= NK_PLUS expr_or_subquery", + /* 381 */ "expression ::= NK_MINUS expr_or_subquery", + /* 382 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 383 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 384 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 385 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 386 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 387 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 388 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 389 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 390 */ "expression_list ::= expr_or_subquery", + /* 391 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 392 */ "column_reference ::= column_name", + /* 393 */ "column_reference ::= table_name NK_DOT column_name", + /* 394 */ "pseudo_column ::= ROWTS", + /* 395 */ "pseudo_column ::= TBNAME", + /* 396 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 397 */ "pseudo_column ::= QSTART", + /* 398 */ "pseudo_column ::= QEND", + /* 399 */ "pseudo_column ::= QDURATION", + /* 400 */ "pseudo_column ::= WSTART", + /* 401 */ "pseudo_column ::= WEND", + /* 402 */ "pseudo_column ::= WDURATION", + /* 403 */ "pseudo_column ::= IROWTS", + /* 404 */ "pseudo_column ::= ISFILLED", + /* 405 */ "pseudo_column ::= QTAGS", + /* 406 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 407 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 408 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 409 */ "function_expression ::= literal_func", + /* 410 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 411 */ "literal_func ::= NOW", + /* 412 */ "noarg_func ::= NOW", + /* 413 */ "noarg_func ::= TODAY", + /* 414 */ "noarg_func ::= TIMEZONE", + /* 415 */ "noarg_func ::= DATABASE", + /* 416 */ "noarg_func ::= CLIENT_VERSION", + /* 417 */ "noarg_func ::= SERVER_VERSION", + /* 418 */ "noarg_func ::= SERVER_STATUS", + /* 419 */ "noarg_func ::= CURRENT_USER", + /* 420 */ "noarg_func ::= USER", + /* 421 */ "star_func ::= COUNT", + /* 422 */ "star_func ::= FIRST", + /* 423 */ "star_func ::= LAST", + /* 424 */ "star_func ::= LAST_ROW", + /* 425 */ "star_func_para_list ::= NK_STAR", + /* 426 */ "star_func_para_list ::= other_para_list", + /* 427 */ "other_para_list ::= star_func_para", + /* 428 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 429 */ "star_func_para ::= expr_or_subquery", + /* 430 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 431 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 432 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 433 */ "when_then_list ::= when_then_expr", + /* 434 */ "when_then_list ::= when_then_list when_then_expr", + /* 435 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 436 */ "case_when_else_opt ::=", + /* 437 */ "case_when_else_opt ::= ELSE common_expression", + /* 438 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 439 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 440 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 441 */ "predicate ::= expr_or_subquery IS NULL", + /* 442 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 443 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 444 */ "compare_op ::= NK_LT", + /* 445 */ "compare_op ::= NK_GT", + /* 446 */ "compare_op ::= NK_LE", + /* 447 */ "compare_op ::= NK_GE", + /* 448 */ "compare_op ::= NK_NE", + /* 449 */ "compare_op ::= NK_EQ", + /* 450 */ "compare_op ::= LIKE", + /* 451 */ "compare_op ::= NOT LIKE", + /* 452 */ "compare_op ::= MATCH", + /* 453 */ "compare_op ::= NMATCH", + /* 454 */ "compare_op ::= CONTAINS", + /* 455 */ "in_op ::= IN", + /* 456 */ "in_op ::= NOT IN", + /* 457 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 458 */ "boolean_value_expression ::= boolean_primary", + /* 459 */ "boolean_value_expression ::= NOT boolean_primary", + /* 460 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 461 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 462 */ "boolean_primary ::= predicate", + /* 463 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 464 */ "common_expression ::= expr_or_subquery", + /* 465 */ "common_expression ::= boolean_value_expression", + /* 466 */ "from_clause_opt ::=", + /* 467 */ "from_clause_opt ::= FROM table_reference_list", + /* 468 */ "table_reference_list ::= table_reference", + /* 469 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 470 */ "table_reference ::= table_primary", + /* 471 */ "table_reference ::= joined_table", + /* 472 */ "table_primary ::= table_name alias_opt", + /* 473 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 474 */ "table_primary ::= subquery alias_opt", + /* 475 */ "table_primary ::= parenthesized_joined_table", + /* 476 */ "alias_opt ::=", + /* 477 */ "alias_opt ::= table_alias", + /* 478 */ "alias_opt ::= AS table_alias", + /* 479 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 480 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 481 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 482 */ "join_type ::=", + /* 483 */ "join_type ::= INNER", + /* 484 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 485 */ "set_quantifier_opt ::=", + /* 486 */ "set_quantifier_opt ::= DISTINCT", + /* 487 */ "set_quantifier_opt ::= ALL", + /* 488 */ "select_list ::= select_item", + /* 489 */ "select_list ::= select_list NK_COMMA select_item", + /* 490 */ "select_item ::= NK_STAR", + /* 491 */ "select_item ::= common_expression", + /* 492 */ "select_item ::= common_expression column_alias", + /* 493 */ "select_item ::= common_expression AS column_alias", + /* 494 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 495 */ "where_clause_opt ::=", + /* 496 */ "where_clause_opt ::= WHERE search_condition", + /* 497 */ "partition_by_clause_opt ::=", + /* 498 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 499 */ "partition_list ::= partition_item", + /* 500 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 501 */ "partition_item ::= expr_or_subquery", + /* 502 */ "partition_item ::= expr_or_subquery column_alias", + /* 503 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 504 */ "twindow_clause_opt ::=", + /* 505 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 506 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 507 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 508 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 509 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 510 */ "sliding_opt ::=", + /* 511 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 512 */ "fill_opt ::=", + /* 513 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 514 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 515 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", + /* 516 */ "fill_mode ::= NONE", + /* 517 */ "fill_mode ::= PREV", + /* 518 */ "fill_mode ::= NULL", + /* 519 */ "fill_mode ::= NULL_F", + /* 520 */ "fill_mode ::= LINEAR", + /* 521 */ "fill_mode ::= NEXT", + /* 522 */ "group_by_clause_opt ::=", + /* 523 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 524 */ "group_by_list ::= expr_or_subquery", + /* 525 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 526 */ "having_clause_opt ::=", + /* 527 */ "having_clause_opt ::= HAVING search_condition", + /* 528 */ "range_opt ::=", + /* 529 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 530 */ "every_opt ::=", + /* 531 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 532 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 533 */ "query_simple ::= query_specification", + /* 534 */ "query_simple ::= union_query_expression", + /* 535 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 536 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 537 */ "query_simple_or_subquery ::= query_simple", + /* 538 */ "query_simple_or_subquery ::= subquery", + /* 539 */ "query_or_subquery ::= query_expression", + /* 540 */ "query_or_subquery ::= subquery", + /* 541 */ "order_by_clause_opt ::=", + /* 542 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 543 */ "slimit_clause_opt ::=", + /* 544 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 545 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 546 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 547 */ "limit_clause_opt ::=", + /* 548 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 549 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 550 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 551 */ "subquery ::= NK_LP query_expression NK_RP", + /* 552 */ "subquery ::= NK_LP subquery NK_RP", + /* 553 */ "search_condition ::= common_expression", + /* 554 */ "sort_specification_list ::= sort_specification", + /* 555 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 556 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 557 */ "ordering_specification_opt ::=", + /* 558 */ "ordering_specification_opt ::= ASC", + /* 559 */ "ordering_specification_opt ::= DESC", + /* 560 */ "null_ordering_opt ::=", + /* 561 */ "null_ordering_opt ::= NULLS FIRST", + /* 562 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2523,195 +2562,199 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 324: /* cmd */ - case 327: /* literal */ - case 340: /* db_options */ - case 342: /* alter_db_options */ - case 348: /* retention */ - case 349: /* full_table_name */ - case 352: /* table_options */ - case 356: /* alter_table_clause */ - case 357: /* alter_table_options */ - case 360: /* signed_literal */ - case 361: /* create_subtable_clause */ - case 364: /* drop_table_clause */ - case 367: /* column_def */ - case 371: /* duration_literal */ - case 372: /* rollup_func_name */ - case 374: /* col_name */ - case 375: /* db_name_cond_opt */ - case 376: /* like_pattern_opt */ - case 377: /* table_name_cond */ - case 378: /* from_db_opt */ - case 380: /* tag_item */ - case 382: /* full_index_name */ - case 383: /* index_options */ - case 386: /* sliding_opt */ - case 387: /* sma_stream_opt */ - case 388: /* func */ - case 389: /* query_or_subquery */ - case 392: /* explain_options */ - case 396: /* stream_options */ - case 397: /* subtable_opt */ - case 398: /* expression */ - case 400: /* where_clause_opt */ - case 401: /* signed */ - case 402: /* literal_func */ - case 405: /* expr_or_subquery */ - case 406: /* pseudo_column */ - case 407: /* column_reference */ - case 408: /* function_expression */ - case 409: /* case_when_expression */ - case 414: /* star_func_para */ - case 416: /* case_when_else_opt */ - case 417: /* common_expression */ - case 418: /* when_then_expr */ - case 419: /* predicate */ - case 422: /* in_predicate_value */ - case 423: /* boolean_value_expression */ - case 424: /* boolean_primary */ - case 425: /* from_clause_opt */ - case 426: /* table_reference_list */ - case 427: /* table_reference */ - case 428: /* table_primary */ - case 429: /* joined_table */ - case 431: /* subquery */ - case 432: /* parenthesized_joined_table */ - case 434: /* search_condition */ - case 435: /* query_specification */ - case 439: /* range_opt */ - case 440: /* every_opt */ - case 441: /* fill_opt */ - case 442: /* twindow_clause_opt */ - case 444: /* having_clause_opt */ - case 445: /* select_item */ - case 447: /* partition_item */ - case 450: /* query_expression */ - case 451: /* query_simple */ - case 453: /* slimit_clause_opt */ - case 454: /* limit_clause_opt */ - case 455: /* union_query_expression */ - case 456: /* query_simple_or_subquery */ - case 458: /* sort_specification */ + case 328: /* cmd */ + case 331: /* literal */ + case 344: /* db_options */ + case 346: /* alter_db_options */ + case 352: /* retention */ + case 353: /* full_table_name */ + case 356: /* table_options */ + case 360: /* alter_table_clause */ + case 361: /* alter_table_options */ + case 364: /* signed_literal */ + case 365: /* create_subtable_clause */ + case 368: /* drop_table_clause */ + case 371: /* column_def */ + case 375: /* duration_literal */ + case 376: /* rollup_func_name */ + case 378: /* col_name */ + case 379: /* db_name_cond_opt */ + case 380: /* like_pattern_opt */ + case 381: /* table_name_cond */ + case 382: /* from_db_opt */ + case 384: /* tag_item */ + case 386: /* full_index_name */ + case 387: /* index_options */ + case 390: /* sliding_opt */ + case 391: /* sma_stream_opt */ + case 392: /* func */ + case 394: /* query_or_subquery */ + case 397: /* explain_options */ + case 398: /* insert_query */ + case 402: /* stream_options */ + case 405: /* subtable_opt */ + case 406: /* expression */ + case 408: /* where_clause_opt */ + case 409: /* signed */ + case 410: /* literal_func */ + case 413: /* expr_or_subquery */ + case 414: /* pseudo_column */ + case 415: /* column_reference */ + case 416: /* function_expression */ + case 417: /* case_when_expression */ + case 422: /* star_func_para */ + case 424: /* case_when_else_opt */ + case 425: /* common_expression */ + case 426: /* when_then_expr */ + case 427: /* predicate */ + case 430: /* in_predicate_value */ + case 431: /* boolean_value_expression */ + case 432: /* boolean_primary */ + case 433: /* from_clause_opt */ + case 434: /* table_reference_list */ + case 435: /* table_reference */ + case 436: /* table_primary */ + case 437: /* joined_table */ + case 439: /* subquery */ + case 440: /* parenthesized_joined_table */ + case 442: /* search_condition */ + case 443: /* query_specification */ + case 447: /* range_opt */ + case 448: /* every_opt */ + case 449: /* fill_opt */ + case 450: /* twindow_clause_opt */ + case 452: /* having_clause_opt */ + case 453: /* select_item */ + case 455: /* partition_item */ + case 458: /* query_expression */ + case 459: /* query_simple */ + case 461: /* slimit_clause_opt */ + case 462: /* limit_clause_opt */ + case 463: /* union_query_expression */ + case 464: /* query_simple_or_subquery */ + case 466: /* sort_specification */ { - nodesDestroyNode((yypminor->yy600)); + nodesDestroyNode((yypminor->yy42)); } break; - case 325: /* account_options */ - case 326: /* alter_account_options */ - case 328: /* alter_account_option */ - case 343: /* speed_opt */ - case 394: /* bufsize_opt */ + case 329: /* account_options */ + case 330: /* alter_account_options */ + case 332: /* alter_account_option */ + case 347: /* speed_opt */ + case 400: /* bufsize_opt */ { } break; - case 329: /* user_name */ - case 332: /* priv_level */ - case 335: /* db_name */ - case 336: /* topic_name */ - case 337: /* dnode_endpoint */ - case 358: /* column_name */ - case 366: /* table_name */ - case 373: /* function_name */ - case 381: /* column_alias */ - case 384: /* index_name */ - case 390: /* cgroup_name */ - case 395: /* stream_name */ - case 404: /* table_alias */ - case 410: /* star_func */ - case 412: /* noarg_func */ - case 430: /* alias_opt */ + case 333: /* user_name */ + case 336: /* priv_level */ + case 339: /* db_name */ + case 340: /* topic_name */ + case 341: /* dnode_endpoint */ + case 362: /* column_name */ + case 370: /* table_name */ + case 377: /* function_name */ + case 385: /* column_alias */ + case 388: /* index_name */ + case 393: /* sma_func_name */ + case 395: /* cgroup_name */ + case 401: /* stream_name */ + case 412: /* table_alias */ + case 418: /* star_func */ + case 420: /* noarg_func */ + case 438: /* alias_opt */ { } break; - case 330: /* sysinfo_opt */ + case 334: /* sysinfo_opt */ { } break; - case 331: /* privileges */ - case 333: /* priv_type_list */ - case 334: /* priv_type */ + case 335: /* privileges */ + case 337: /* priv_type_list */ + case 338: /* priv_type */ { } break; - case 338: /* force_opt */ - case 339: /* not_exists_opt */ - case 341: /* exists_opt */ - case 391: /* analyze_opt */ - case 393: /* agg_func_opt */ - case 436: /* set_quantifier_opt */ + case 342: /* force_opt */ + case 343: /* not_exists_opt */ + case 345: /* exists_opt */ + case 396: /* analyze_opt */ + case 399: /* agg_func_opt */ + case 444: /* set_quantifier_opt */ { } break; - case 344: /* integer_list */ - case 345: /* variable_list */ - case 346: /* retention_list */ - case 350: /* column_def_list */ - case 351: /* tags_def_opt */ - case 353: /* multi_create_clause */ - case 354: /* tags_def */ - case 355: /* multi_drop_clause */ - case 362: /* specific_cols_opt */ - case 363: /* expression_list */ - case 365: /* col_name_list */ - case 368: /* duration_list */ - case 369: /* rollup_func_list */ - case 379: /* tag_list_opt */ - case 385: /* func_list */ - case 399: /* dnode_list */ - case 403: /* literal_list */ - case 411: /* star_func_para_list */ - case 413: /* other_para_list */ - case 415: /* when_then_list */ - case 437: /* select_list */ - case 438: /* partition_by_clause_opt */ - case 443: /* group_by_clause_opt */ - case 446: /* partition_list */ - case 449: /* group_by_list */ - case 452: /* order_by_clause_opt */ - case 457: /* sort_specification_list */ + case 348: /* integer_list */ + case 349: /* variable_list */ + case 350: /* retention_list */ + case 354: /* column_def_list */ + case 355: /* tags_def_opt */ + case 357: /* multi_create_clause */ + case 358: /* tags_def */ + case 359: /* multi_drop_clause */ + case 366: /* specific_cols_opt */ + case 367: /* expression_list */ + case 369: /* col_name_list */ + case 372: /* duration_list */ + case 373: /* rollup_func_list */ + case 383: /* tag_list_opt */ + case 389: /* func_list */ + case 403: /* col_list_opt */ + case 404: /* tag_def_or_ref_opt */ + case 407: /* dnode_list */ + case 411: /* literal_list */ + case 419: /* star_func_para_list */ + case 421: /* other_para_list */ + case 423: /* when_then_list */ + case 445: /* select_list */ + case 446: /* partition_by_clause_opt */ + case 451: /* group_by_clause_opt */ + case 454: /* partition_list */ + case 457: /* group_by_list */ + case 460: /* order_by_clause_opt */ + case 465: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy601)); + nodesDestroyList((yypminor->yy110)); } break; - case 347: /* alter_db_option */ - case 370: /* alter_table_option */ + case 351: /* alter_db_option */ + case 374: /* alter_table_option */ { } break; - case 359: /* type_name */ + case 363: /* type_name */ { } break; - case 420: /* compare_op */ - case 421: /* in_op */ + case 428: /* compare_op */ + case 429: /* in_op */ { } break; - case 433: /* join_type */ + case 441: /* join_type */ { } break; - case 448: /* fill_mode */ + case 456: /* fill_mode */ { } break; - case 459: /* ordering_specification_opt */ + case 467: /* ordering_specification_opt */ { } break; - case 460: /* null_ordering_opt */ + case 468: /* null_ordering_opt */ { } @@ -3010,551 +3053,569 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 324, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 324, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 325, 0 }, /* (2) account_options ::= */ - { 325, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 325, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 325, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 325, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 325, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 325, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 325, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 325, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 325, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 326, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 326, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 328, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 328, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 328, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 328, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 328, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 328, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 328, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 328, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 328, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 328, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 324, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - { 324, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 324, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - { 324, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - { 324, -3 }, /* (28) cmd ::= DROP USER user_name */ - { 330, 0 }, /* (29) sysinfo_opt ::= */ - { 330, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 324, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 324, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 331, -1 }, /* (33) privileges ::= ALL */ - { 331, -1 }, /* (34) privileges ::= priv_type_list */ - { 331, -1 }, /* (35) privileges ::= SUBSCRIBE */ - { 333, -1 }, /* (36) priv_type_list ::= priv_type */ - { 333, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 334, -1 }, /* (38) priv_type ::= READ */ - { 334, -1 }, /* (39) priv_type ::= WRITE */ - { 332, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 332, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - { 332, -1 }, /* (42) priv_level ::= topic_name */ - { 324, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ - { 324, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 324, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ - { 324, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ - { 324, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 324, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 324, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ - { 324, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 337, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ - { 337, -1 }, /* (52) dnode_endpoint ::= NK_ID */ - { 337, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ - { 338, 0 }, /* (54) force_opt ::= */ - { 338, -1 }, /* (55) force_opt ::= FORCE */ - { 324, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ - { 324, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 324, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 324, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 324, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ - { 324, -2 }, /* (68) cmd ::= USE db_name */ - { 324, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 324, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ - { 324, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ - { 339, -3 }, /* (72) not_exists_opt ::= IF NOT EXISTS */ - { 339, 0 }, /* (73) not_exists_opt ::= */ - { 341, -2 }, /* (74) exists_opt ::= IF EXISTS */ - { 341, 0 }, /* (75) exists_opt ::= */ - { 340, 0 }, /* (76) db_options ::= */ - { 340, -3 }, /* (77) db_options ::= db_options BUFFER NK_INTEGER */ - { 340, -3 }, /* (78) db_options ::= db_options CACHEMODEL NK_STRING */ - { 340, -3 }, /* (79) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 340, -3 }, /* (80) db_options ::= db_options COMP NK_INTEGER */ - { 340, -3 }, /* (81) db_options ::= db_options DURATION NK_INTEGER */ - { 340, -3 }, /* (82) db_options ::= db_options DURATION NK_VARIABLE */ - { 340, -3 }, /* (83) db_options ::= db_options MAXROWS NK_INTEGER */ - { 340, -3 }, /* (84) db_options ::= db_options MINROWS NK_INTEGER */ - { 340, -3 }, /* (85) db_options ::= db_options KEEP integer_list */ - { 340, -3 }, /* (86) db_options ::= db_options KEEP variable_list */ - { 340, -3 }, /* (87) db_options ::= db_options PAGES NK_INTEGER */ - { 340, -3 }, /* (88) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 340, -3 }, /* (89) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - { 340, -3 }, /* (90) db_options ::= db_options PRECISION NK_STRING */ - { 340, -3 }, /* (91) db_options ::= db_options REPLICA NK_INTEGER */ - { 340, -3 }, /* (92) db_options ::= db_options VGROUPS NK_INTEGER */ - { 340, -3 }, /* (93) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 340, -3 }, /* (94) db_options ::= db_options RETENTIONS retention_list */ - { 340, -3 }, /* (95) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 340, -3 }, /* (96) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 340, -3 }, /* (97) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 340, -3 }, /* (98) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 340, -4 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 340, -3 }, /* (100) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 340, -4 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 340, -3 }, /* (102) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 340, -3 }, /* (103) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 340, -3 }, /* (104) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - { 340, -3 }, /* (105) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ - { 340, -3 }, /* (106) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ - { 342, -1 }, /* (107) alter_db_options ::= alter_db_option */ - { 342, -2 }, /* (108) alter_db_options ::= alter_db_options alter_db_option */ - { 347, -2 }, /* (109) alter_db_option ::= BUFFER NK_INTEGER */ - { 347, -2 }, /* (110) alter_db_option ::= CACHEMODEL NK_STRING */ - { 347, -2 }, /* (111) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 347, -2 }, /* (112) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 347, -2 }, /* (113) alter_db_option ::= KEEP integer_list */ - { 347, -2 }, /* (114) alter_db_option ::= KEEP variable_list */ - { 347, -2 }, /* (115) alter_db_option ::= PAGES NK_INTEGER */ - { 347, -2 }, /* (116) alter_db_option ::= REPLICA NK_INTEGER */ - { 347, -2 }, /* (117) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 347, -2 }, /* (118) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - { 344, -1 }, /* (119) integer_list ::= NK_INTEGER */ - { 344, -3 }, /* (120) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 345, -1 }, /* (121) variable_list ::= NK_VARIABLE */ - { 345, -3 }, /* (122) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 346, -1 }, /* (123) retention_list ::= retention */ - { 346, -3 }, /* (124) retention_list ::= retention_list NK_COMMA retention */ - { 348, -3 }, /* (125) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 343, 0 }, /* (126) speed_opt ::= */ - { 343, -2 }, /* (127) speed_opt ::= MAX_SPEED NK_INTEGER */ - { 324, -9 }, /* (128) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 324, -3 }, /* (129) cmd ::= CREATE TABLE multi_create_clause */ - { 324, -9 }, /* (130) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 324, -3 }, /* (131) cmd ::= DROP TABLE multi_drop_clause */ - { 324, -4 }, /* (132) cmd ::= DROP STABLE exists_opt full_table_name */ - { 324, -3 }, /* (133) cmd ::= ALTER TABLE alter_table_clause */ - { 324, -3 }, /* (134) cmd ::= ALTER STABLE alter_table_clause */ - { 356, -2 }, /* (135) alter_table_clause ::= full_table_name alter_table_options */ - { 356, -5 }, /* (136) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 356, -4 }, /* (137) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 356, -5 }, /* (138) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 356, -5 }, /* (139) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 356, -5 }, /* (140) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 356, -4 }, /* (141) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 356, -5 }, /* (142) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 356, -5 }, /* (143) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 356, -6 }, /* (144) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 353, -1 }, /* (145) multi_create_clause ::= create_subtable_clause */ - { 353, -2 }, /* (146) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 361, -10 }, /* (147) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 355, -1 }, /* (148) multi_drop_clause ::= drop_table_clause */ - { 355, -2 }, /* (149) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 364, -2 }, /* (150) drop_table_clause ::= exists_opt full_table_name */ - { 362, 0 }, /* (151) specific_cols_opt ::= */ - { 362, -3 }, /* (152) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 349, -1 }, /* (153) full_table_name ::= table_name */ - { 349, -3 }, /* (154) full_table_name ::= db_name NK_DOT table_name */ - { 350, -1 }, /* (155) column_def_list ::= column_def */ - { 350, -3 }, /* (156) column_def_list ::= column_def_list NK_COMMA column_def */ - { 367, -2 }, /* (157) column_def ::= column_name type_name */ - { 367, -4 }, /* (158) column_def ::= column_name type_name COMMENT NK_STRING */ - { 359, -1 }, /* (159) type_name ::= BOOL */ - { 359, -1 }, /* (160) type_name ::= TINYINT */ - { 359, -1 }, /* (161) type_name ::= SMALLINT */ - { 359, -1 }, /* (162) type_name ::= INT */ - { 359, -1 }, /* (163) type_name ::= INTEGER */ - { 359, -1 }, /* (164) type_name ::= BIGINT */ - { 359, -1 }, /* (165) type_name ::= FLOAT */ - { 359, -1 }, /* (166) type_name ::= DOUBLE */ - { 359, -4 }, /* (167) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 359, -1 }, /* (168) type_name ::= TIMESTAMP */ - { 359, -4 }, /* (169) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 359, -2 }, /* (170) type_name ::= TINYINT UNSIGNED */ - { 359, -2 }, /* (171) type_name ::= SMALLINT UNSIGNED */ - { 359, -2 }, /* (172) type_name ::= INT UNSIGNED */ - { 359, -2 }, /* (173) type_name ::= BIGINT UNSIGNED */ - { 359, -1 }, /* (174) type_name ::= JSON */ - { 359, -4 }, /* (175) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 359, -1 }, /* (176) type_name ::= MEDIUMBLOB */ - { 359, -1 }, /* (177) type_name ::= BLOB */ - { 359, -4 }, /* (178) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 359, -1 }, /* (179) type_name ::= DECIMAL */ - { 359, -4 }, /* (180) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 359, -6 }, /* (181) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 351, 0 }, /* (182) tags_def_opt ::= */ - { 351, -1 }, /* (183) tags_def_opt ::= tags_def */ - { 354, -4 }, /* (184) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 352, 0 }, /* (185) table_options ::= */ - { 352, -3 }, /* (186) table_options ::= table_options COMMENT NK_STRING */ - { 352, -3 }, /* (187) table_options ::= table_options MAX_DELAY duration_list */ - { 352, -3 }, /* (188) table_options ::= table_options WATERMARK duration_list */ - { 352, -5 }, /* (189) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 352, -3 }, /* (190) table_options ::= table_options TTL NK_INTEGER */ - { 352, -5 }, /* (191) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 352, -3 }, /* (192) table_options ::= table_options DELETE_MARK duration_list */ - { 357, -1 }, /* (193) alter_table_options ::= alter_table_option */ - { 357, -2 }, /* (194) alter_table_options ::= alter_table_options alter_table_option */ - { 370, -2 }, /* (195) alter_table_option ::= COMMENT NK_STRING */ - { 370, -2 }, /* (196) alter_table_option ::= TTL NK_INTEGER */ - { 368, -1 }, /* (197) duration_list ::= duration_literal */ - { 368, -3 }, /* (198) duration_list ::= duration_list NK_COMMA duration_literal */ - { 369, -1 }, /* (199) rollup_func_list ::= rollup_func_name */ - { 369, -3 }, /* (200) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 372, -1 }, /* (201) rollup_func_name ::= function_name */ - { 372, -1 }, /* (202) rollup_func_name ::= FIRST */ - { 372, -1 }, /* (203) rollup_func_name ::= LAST */ - { 365, -1 }, /* (204) col_name_list ::= col_name */ - { 365, -3 }, /* (205) col_name_list ::= col_name_list NK_COMMA col_name */ - { 374, -1 }, /* (206) col_name ::= column_name */ - { 324, -2 }, /* (207) cmd ::= SHOW DNODES */ - { 324, -2 }, /* (208) cmd ::= SHOW USERS */ - { 324, -3 }, /* (209) cmd ::= SHOW USER PRIVILEGES */ - { 324, -2 }, /* (210) cmd ::= SHOW DATABASES */ - { 324, -4 }, /* (211) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 324, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 324, -3 }, /* (213) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 324, -2 }, /* (214) cmd ::= SHOW MNODES */ - { 324, -2 }, /* (215) cmd ::= SHOW QNODES */ - { 324, -2 }, /* (216) cmd ::= SHOW FUNCTIONS */ - { 324, -5 }, /* (217) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 324, -2 }, /* (218) cmd ::= SHOW STREAMS */ - { 324, -2 }, /* (219) cmd ::= SHOW ACCOUNTS */ - { 324, -2 }, /* (220) cmd ::= SHOW APPS */ - { 324, -2 }, /* (221) cmd ::= SHOW CONNECTIONS */ - { 324, -2 }, /* (222) cmd ::= SHOW LICENCES */ - { 324, -2 }, /* (223) cmd ::= SHOW GRANTS */ - { 324, -4 }, /* (224) cmd ::= SHOW CREATE DATABASE db_name */ - { 324, -4 }, /* (225) cmd ::= SHOW CREATE TABLE full_table_name */ - { 324, -4 }, /* (226) cmd ::= SHOW CREATE STABLE full_table_name */ - { 324, -2 }, /* (227) cmd ::= SHOW QUERIES */ - { 324, -2 }, /* (228) cmd ::= SHOW SCORES */ - { 324, -2 }, /* (229) cmd ::= SHOW TOPICS */ - { 324, -2 }, /* (230) cmd ::= SHOW VARIABLES */ - { 324, -3 }, /* (231) cmd ::= SHOW CLUSTER VARIABLES */ - { 324, -3 }, /* (232) cmd ::= SHOW LOCAL VARIABLES */ - { 324, -5 }, /* (233) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 324, -2 }, /* (234) cmd ::= SHOW BNODES */ - { 324, -2 }, /* (235) cmd ::= SHOW SNODES */ - { 324, -2 }, /* (236) cmd ::= SHOW CLUSTER */ - { 324, -2 }, /* (237) cmd ::= SHOW TRANSACTIONS */ - { 324, -4 }, /* (238) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 324, -2 }, /* (239) cmd ::= SHOW CONSUMERS */ - { 324, -2 }, /* (240) cmd ::= SHOW SUBSCRIPTIONS */ - { 324, -5 }, /* (241) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 324, -7 }, /* (242) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 324, -3 }, /* (243) cmd ::= SHOW VNODES NK_INTEGER */ - { 324, -3 }, /* (244) cmd ::= SHOW VNODES NK_STRING */ - { 375, 0 }, /* (245) db_name_cond_opt ::= */ - { 375, -2 }, /* (246) db_name_cond_opt ::= db_name NK_DOT */ - { 376, 0 }, /* (247) like_pattern_opt ::= */ - { 376, -2 }, /* (248) like_pattern_opt ::= LIKE NK_STRING */ - { 377, -1 }, /* (249) table_name_cond ::= table_name */ - { 378, 0 }, /* (250) from_db_opt ::= */ - { 378, -2 }, /* (251) from_db_opt ::= FROM db_name */ - { 379, 0 }, /* (252) tag_list_opt ::= */ - { 379, -1 }, /* (253) tag_list_opt ::= tag_item */ - { 379, -3 }, /* (254) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 380, -1 }, /* (255) tag_item ::= TBNAME */ - { 380, -1 }, /* (256) tag_item ::= QTAGS */ - { 380, -1 }, /* (257) tag_item ::= column_name */ - { 380, -2 }, /* (258) tag_item ::= column_name column_alias */ - { 380, -3 }, /* (259) tag_item ::= column_name AS column_alias */ - { 324, -8 }, /* (260) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - { 324, -4 }, /* (261) cmd ::= DROP INDEX exists_opt full_index_name */ - { 382, -1 }, /* (262) full_index_name ::= index_name */ - { 382, -3 }, /* (263) full_index_name ::= db_name NK_DOT index_name */ - { 383, -10 }, /* (264) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 383, -12 }, /* (265) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 385, -1 }, /* (266) func_list ::= func */ - { 385, -3 }, /* (267) func_list ::= func_list NK_COMMA func */ - { 388, -4 }, /* (268) func ::= function_name NK_LP expression_list NK_RP */ - { 387, 0 }, /* (269) sma_stream_opt ::= */ - { 387, -3 }, /* (270) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - { 387, -3 }, /* (271) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - { 387, -3 }, /* (272) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - { 324, -6 }, /* (273) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 324, -7 }, /* (274) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 324, -9 }, /* (275) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 324, -7 }, /* (276) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 324, -9 }, /* (277) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 324, -4 }, /* (278) cmd ::= DROP TOPIC exists_opt topic_name */ - { 324, -7 }, /* (279) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 324, -2 }, /* (280) cmd ::= DESC full_table_name */ - { 324, -2 }, /* (281) cmd ::= DESCRIBE full_table_name */ - { 324, -3 }, /* (282) cmd ::= RESET QUERY CACHE */ - { 324, -4 }, /* (283) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 391, 0 }, /* (284) analyze_opt ::= */ - { 391, -1 }, /* (285) analyze_opt ::= ANALYZE */ - { 392, 0 }, /* (286) explain_options ::= */ - { 392, -3 }, /* (287) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 392, -3 }, /* (288) explain_options ::= explain_options RATIO NK_FLOAT */ - { 324, -10 }, /* (289) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 324, -4 }, /* (290) cmd ::= DROP FUNCTION exists_opt function_name */ - { 393, 0 }, /* (291) agg_func_opt ::= */ - { 393, -1 }, /* (292) agg_func_opt ::= AGGREGATE */ - { 394, 0 }, /* (293) bufsize_opt ::= */ - { 394, -2 }, /* (294) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 324, -11 }, /* (295) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ - { 324, -4 }, /* (296) cmd ::= DROP STREAM exists_opt stream_name */ - { 396, 0 }, /* (297) stream_options ::= */ - { 396, -3 }, /* (298) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 396, -3 }, /* (299) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 396, -4 }, /* (300) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 396, -3 }, /* (301) stream_options ::= stream_options WATERMARK duration_literal */ - { 396, -4 }, /* (302) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 396, -3 }, /* (303) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 396, -3 }, /* (304) stream_options ::= stream_options DELETE_MARK duration_literal */ - { 396, -4 }, /* (305) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - { 397, 0 }, /* (306) subtable_opt ::= */ - { 397, -4 }, /* (307) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 324, -3 }, /* (308) cmd ::= KILL CONNECTION NK_INTEGER */ - { 324, -3 }, /* (309) cmd ::= KILL QUERY NK_STRING */ - { 324, -3 }, /* (310) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 324, -2 }, /* (311) cmd ::= BALANCE VGROUP */ - { 324, -4 }, /* (312) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 324, -4 }, /* (313) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 324, -3 }, /* (314) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 399, -2 }, /* (315) dnode_list ::= DNODE NK_INTEGER */ - { 399, -3 }, /* (316) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 324, -4 }, /* (317) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 324, -1 }, /* (318) cmd ::= query_or_subquery */ - { 324, -7 }, /* (319) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 324, -4 }, /* (320) cmd ::= INSERT INTO full_table_name query_or_subquery */ - { 327, -1 }, /* (321) literal ::= NK_INTEGER */ - { 327, -1 }, /* (322) literal ::= NK_FLOAT */ - { 327, -1 }, /* (323) literal ::= NK_STRING */ - { 327, -1 }, /* (324) literal ::= NK_BOOL */ - { 327, -2 }, /* (325) literal ::= TIMESTAMP NK_STRING */ - { 327, -1 }, /* (326) literal ::= duration_literal */ - { 327, -1 }, /* (327) literal ::= NULL */ - { 327, -1 }, /* (328) literal ::= NK_QUESTION */ - { 371, -1 }, /* (329) duration_literal ::= NK_VARIABLE */ - { 401, -1 }, /* (330) signed ::= NK_INTEGER */ - { 401, -2 }, /* (331) signed ::= NK_PLUS NK_INTEGER */ - { 401, -2 }, /* (332) signed ::= NK_MINUS NK_INTEGER */ - { 401, -1 }, /* (333) signed ::= NK_FLOAT */ - { 401, -2 }, /* (334) signed ::= NK_PLUS NK_FLOAT */ - { 401, -2 }, /* (335) signed ::= NK_MINUS NK_FLOAT */ - { 360, -1 }, /* (336) signed_literal ::= signed */ - { 360, -1 }, /* (337) signed_literal ::= NK_STRING */ - { 360, -1 }, /* (338) signed_literal ::= NK_BOOL */ - { 360, -2 }, /* (339) signed_literal ::= TIMESTAMP NK_STRING */ - { 360, -1 }, /* (340) signed_literal ::= duration_literal */ - { 360, -1 }, /* (341) signed_literal ::= NULL */ - { 360, -1 }, /* (342) signed_literal ::= literal_func */ - { 360, -1 }, /* (343) signed_literal ::= NK_QUESTION */ - { 403, -1 }, /* (344) literal_list ::= signed_literal */ - { 403, -3 }, /* (345) literal_list ::= literal_list NK_COMMA signed_literal */ - { 335, -1 }, /* (346) db_name ::= NK_ID */ - { 366, -1 }, /* (347) table_name ::= NK_ID */ - { 358, -1 }, /* (348) column_name ::= NK_ID */ - { 373, -1 }, /* (349) function_name ::= NK_ID */ - { 404, -1 }, /* (350) table_alias ::= NK_ID */ - { 381, -1 }, /* (351) column_alias ::= NK_ID */ - { 329, -1 }, /* (352) user_name ::= NK_ID */ - { 336, -1 }, /* (353) topic_name ::= NK_ID */ - { 395, -1 }, /* (354) stream_name ::= NK_ID */ - { 390, -1 }, /* (355) cgroup_name ::= NK_ID */ - { 384, -1 }, /* (356) index_name ::= NK_ID */ - { 405, -1 }, /* (357) expr_or_subquery ::= expression */ - { 398, -1 }, /* (358) expression ::= literal */ - { 398, -1 }, /* (359) expression ::= pseudo_column */ - { 398, -1 }, /* (360) expression ::= column_reference */ - { 398, -1 }, /* (361) expression ::= function_expression */ - { 398, -1 }, /* (362) expression ::= case_when_expression */ - { 398, -3 }, /* (363) expression ::= NK_LP expression NK_RP */ - { 398, -2 }, /* (364) expression ::= NK_PLUS expr_or_subquery */ - { 398, -2 }, /* (365) expression ::= NK_MINUS expr_or_subquery */ - { 398, -3 }, /* (366) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 398, -3 }, /* (367) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 398, -3 }, /* (368) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 398, -3 }, /* (369) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 398, -3 }, /* (370) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 398, -3 }, /* (371) expression ::= column_reference NK_ARROW NK_STRING */ - { 398, -3 }, /* (372) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 398, -3 }, /* (373) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 363, -1 }, /* (374) expression_list ::= expr_or_subquery */ - { 363, -3 }, /* (375) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 407, -1 }, /* (376) column_reference ::= column_name */ - { 407, -3 }, /* (377) column_reference ::= table_name NK_DOT column_name */ - { 406, -1 }, /* (378) pseudo_column ::= ROWTS */ - { 406, -1 }, /* (379) pseudo_column ::= TBNAME */ - { 406, -3 }, /* (380) pseudo_column ::= table_name NK_DOT TBNAME */ - { 406, -1 }, /* (381) pseudo_column ::= QSTART */ - { 406, -1 }, /* (382) pseudo_column ::= QEND */ - { 406, -1 }, /* (383) pseudo_column ::= QDURATION */ - { 406, -1 }, /* (384) pseudo_column ::= WSTART */ - { 406, -1 }, /* (385) pseudo_column ::= WEND */ - { 406, -1 }, /* (386) pseudo_column ::= WDURATION */ - { 406, -1 }, /* (387) pseudo_column ::= IROWTS */ - { 406, -1 }, /* (388) pseudo_column ::= QTAGS */ - { 408, -4 }, /* (389) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 408, -4 }, /* (390) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 408, -6 }, /* (391) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 408, -1 }, /* (392) function_expression ::= literal_func */ - { 402, -3 }, /* (393) literal_func ::= noarg_func NK_LP NK_RP */ - { 402, -1 }, /* (394) literal_func ::= NOW */ - { 412, -1 }, /* (395) noarg_func ::= NOW */ - { 412, -1 }, /* (396) noarg_func ::= TODAY */ - { 412, -1 }, /* (397) noarg_func ::= TIMEZONE */ - { 412, -1 }, /* (398) noarg_func ::= DATABASE */ - { 412, -1 }, /* (399) noarg_func ::= CLIENT_VERSION */ - { 412, -1 }, /* (400) noarg_func ::= SERVER_VERSION */ - { 412, -1 }, /* (401) noarg_func ::= SERVER_STATUS */ - { 412, -1 }, /* (402) noarg_func ::= CURRENT_USER */ - { 412, -1 }, /* (403) noarg_func ::= USER */ - { 410, -1 }, /* (404) star_func ::= COUNT */ - { 410, -1 }, /* (405) star_func ::= FIRST */ - { 410, -1 }, /* (406) star_func ::= LAST */ - { 410, -1 }, /* (407) star_func ::= LAST_ROW */ - { 411, -1 }, /* (408) star_func_para_list ::= NK_STAR */ - { 411, -1 }, /* (409) star_func_para_list ::= other_para_list */ - { 413, -1 }, /* (410) other_para_list ::= star_func_para */ - { 413, -3 }, /* (411) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 414, -1 }, /* (412) star_func_para ::= expr_or_subquery */ - { 414, -3 }, /* (413) star_func_para ::= table_name NK_DOT NK_STAR */ - { 409, -4 }, /* (414) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 409, -5 }, /* (415) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 415, -1 }, /* (416) when_then_list ::= when_then_expr */ - { 415, -2 }, /* (417) when_then_list ::= when_then_list when_then_expr */ - { 418, -4 }, /* (418) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 416, 0 }, /* (419) case_when_else_opt ::= */ - { 416, -2 }, /* (420) case_when_else_opt ::= ELSE common_expression */ - { 419, -3 }, /* (421) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 419, -5 }, /* (422) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 419, -6 }, /* (423) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 419, -3 }, /* (424) predicate ::= expr_or_subquery IS NULL */ - { 419, -4 }, /* (425) predicate ::= expr_or_subquery IS NOT NULL */ - { 419, -3 }, /* (426) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 420, -1 }, /* (427) compare_op ::= NK_LT */ - { 420, -1 }, /* (428) compare_op ::= NK_GT */ - { 420, -1 }, /* (429) compare_op ::= NK_LE */ - { 420, -1 }, /* (430) compare_op ::= NK_GE */ - { 420, -1 }, /* (431) compare_op ::= NK_NE */ - { 420, -1 }, /* (432) compare_op ::= NK_EQ */ - { 420, -1 }, /* (433) compare_op ::= LIKE */ - { 420, -2 }, /* (434) compare_op ::= NOT LIKE */ - { 420, -1 }, /* (435) compare_op ::= MATCH */ - { 420, -1 }, /* (436) compare_op ::= NMATCH */ - { 420, -1 }, /* (437) compare_op ::= CONTAINS */ - { 421, -1 }, /* (438) in_op ::= IN */ - { 421, -2 }, /* (439) in_op ::= NOT IN */ - { 422, -3 }, /* (440) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 423, -1 }, /* (441) boolean_value_expression ::= boolean_primary */ - { 423, -2 }, /* (442) boolean_value_expression ::= NOT boolean_primary */ - { 423, -3 }, /* (443) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 423, -3 }, /* (444) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 424, -1 }, /* (445) boolean_primary ::= predicate */ - { 424, -3 }, /* (446) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 417, -1 }, /* (447) common_expression ::= expr_or_subquery */ - { 417, -1 }, /* (448) common_expression ::= boolean_value_expression */ - { 425, 0 }, /* (449) from_clause_opt ::= */ - { 425, -2 }, /* (450) from_clause_opt ::= FROM table_reference_list */ - { 426, -1 }, /* (451) table_reference_list ::= table_reference */ - { 426, -3 }, /* (452) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 427, -1 }, /* (453) table_reference ::= table_primary */ - { 427, -1 }, /* (454) table_reference ::= joined_table */ - { 428, -2 }, /* (455) table_primary ::= table_name alias_opt */ - { 428, -4 }, /* (456) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 428, -2 }, /* (457) table_primary ::= subquery alias_opt */ - { 428, -1 }, /* (458) table_primary ::= parenthesized_joined_table */ - { 430, 0 }, /* (459) alias_opt ::= */ - { 430, -1 }, /* (460) alias_opt ::= table_alias */ - { 430, -2 }, /* (461) alias_opt ::= AS table_alias */ - { 432, -3 }, /* (462) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 432, -3 }, /* (463) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 429, -6 }, /* (464) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 433, 0 }, /* (465) join_type ::= */ - { 433, -1 }, /* (466) join_type ::= INNER */ - { 435, -12 }, /* (467) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 436, 0 }, /* (468) set_quantifier_opt ::= */ - { 436, -1 }, /* (469) set_quantifier_opt ::= DISTINCT */ - { 436, -1 }, /* (470) set_quantifier_opt ::= ALL */ - { 437, -1 }, /* (471) select_list ::= select_item */ - { 437, -3 }, /* (472) select_list ::= select_list NK_COMMA select_item */ - { 445, -1 }, /* (473) select_item ::= NK_STAR */ - { 445, -1 }, /* (474) select_item ::= common_expression */ - { 445, -2 }, /* (475) select_item ::= common_expression column_alias */ - { 445, -3 }, /* (476) select_item ::= common_expression AS column_alias */ - { 445, -3 }, /* (477) select_item ::= table_name NK_DOT NK_STAR */ - { 400, 0 }, /* (478) where_clause_opt ::= */ - { 400, -2 }, /* (479) where_clause_opt ::= WHERE search_condition */ - { 438, 0 }, /* (480) partition_by_clause_opt ::= */ - { 438, -3 }, /* (481) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 446, -1 }, /* (482) partition_list ::= partition_item */ - { 446, -3 }, /* (483) partition_list ::= partition_list NK_COMMA partition_item */ - { 447, -1 }, /* (484) partition_item ::= expr_or_subquery */ - { 447, -2 }, /* (485) partition_item ::= expr_or_subquery column_alias */ - { 447, -3 }, /* (486) partition_item ::= expr_or_subquery AS column_alias */ - { 442, 0 }, /* (487) twindow_clause_opt ::= */ - { 442, -6 }, /* (488) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 442, -4 }, /* (489) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 442, -6 }, /* (490) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 442, -8 }, /* (491) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 386, 0 }, /* (492) sliding_opt ::= */ - { 386, -4 }, /* (493) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 441, 0 }, /* (494) fill_opt ::= */ - { 441, -4 }, /* (495) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 441, -6 }, /* (496) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 441, -6 }, /* (497) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ - { 448, -1 }, /* (498) fill_mode ::= NONE */ - { 448, -1 }, /* (499) fill_mode ::= PREV */ - { 448, -1 }, /* (500) fill_mode ::= NULL */ - { 448, -1 }, /* (501) fill_mode ::= NULL_F */ - { 448, -1 }, /* (502) fill_mode ::= LINEAR */ - { 448, -1 }, /* (503) fill_mode ::= NEXT */ - { 443, 0 }, /* (504) group_by_clause_opt ::= */ - { 443, -3 }, /* (505) group_by_clause_opt ::= GROUP BY group_by_list */ - { 449, -1 }, /* (506) group_by_list ::= expr_or_subquery */ - { 449, -3 }, /* (507) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 444, 0 }, /* (508) having_clause_opt ::= */ - { 444, -2 }, /* (509) having_clause_opt ::= HAVING search_condition */ - { 439, 0 }, /* (510) range_opt ::= */ - { 439, -6 }, /* (511) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 440, 0 }, /* (512) every_opt ::= */ - { 440, -4 }, /* (513) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 450, -4 }, /* (514) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 451, -1 }, /* (515) query_simple ::= query_specification */ - { 451, -1 }, /* (516) query_simple ::= union_query_expression */ - { 455, -4 }, /* (517) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 455, -3 }, /* (518) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 456, -1 }, /* (519) query_simple_or_subquery ::= query_simple */ - { 456, -1 }, /* (520) query_simple_or_subquery ::= subquery */ - { 389, -1 }, /* (521) query_or_subquery ::= query_expression */ - { 389, -1 }, /* (522) query_or_subquery ::= subquery */ - { 452, 0 }, /* (523) order_by_clause_opt ::= */ - { 452, -3 }, /* (524) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 453, 0 }, /* (525) slimit_clause_opt ::= */ - { 453, -2 }, /* (526) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 453, -4 }, /* (527) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 453, -4 }, /* (528) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 454, 0 }, /* (529) limit_clause_opt ::= */ - { 454, -2 }, /* (530) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 454, -4 }, /* (531) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 454, -4 }, /* (532) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 431, -3 }, /* (533) subquery ::= NK_LP query_expression NK_RP */ - { 431, -3 }, /* (534) subquery ::= NK_LP subquery NK_RP */ - { 434, -1 }, /* (535) search_condition ::= common_expression */ - { 457, -1 }, /* (536) sort_specification_list ::= sort_specification */ - { 457, -3 }, /* (537) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 458, -3 }, /* (538) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 459, 0 }, /* (539) ordering_specification_opt ::= */ - { 459, -1 }, /* (540) ordering_specification_opt ::= ASC */ - { 459, -1 }, /* (541) ordering_specification_opt ::= DESC */ - { 460, 0 }, /* (542) null_ordering_opt ::= */ - { 460, -2 }, /* (543) null_ordering_opt ::= NULLS FIRST */ - { 460, -2 }, /* (544) null_ordering_opt ::= NULLS LAST */ + { 328, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 328, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 329, 0 }, /* (2) account_options ::= */ + { 329, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 329, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 329, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 329, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 329, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 329, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 329, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 329, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 329, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 330, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 330, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 332, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 332, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 332, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 332, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 332, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 332, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 332, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 332, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 332, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 332, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 328, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 328, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 328, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 328, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 328, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 334, 0 }, /* (29) sysinfo_opt ::= */ + { 334, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 328, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ + { 328, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 335, -1 }, /* (33) privileges ::= ALL */ + { 335, -1 }, /* (34) privileges ::= priv_type_list */ + { 335, -1 }, /* (35) privileges ::= SUBSCRIBE */ + { 337, -1 }, /* (36) priv_type_list ::= priv_type */ + { 337, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 338, -1 }, /* (38) priv_type ::= READ */ + { 338, -1 }, /* (39) priv_type ::= WRITE */ + { 336, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 336, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ + { 336, -1 }, /* (42) priv_level ::= topic_name */ + { 328, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ + { 328, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 328, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ + { 328, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ + { 328, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 328, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 328, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ + { 328, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 341, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ + { 341, -1 }, /* (52) dnode_endpoint ::= NK_ID */ + { 341, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ + { 342, 0 }, /* (54) force_opt ::= */ + { 342, -1 }, /* (55) force_opt ::= FORCE */ + { 328, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ + { 328, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 328, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 328, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 328, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ + { 328, -2 }, /* (68) cmd ::= USE db_name */ + { 328, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 328, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ + { 328, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ + { 328, -3 }, /* (72) cmd ::= COMPACT DATABASE db_name */ + { 343, -3 }, /* (73) not_exists_opt ::= IF NOT EXISTS */ + { 343, 0 }, /* (74) not_exists_opt ::= */ + { 345, -2 }, /* (75) exists_opt ::= IF EXISTS */ + { 345, 0 }, /* (76) exists_opt ::= */ + { 344, 0 }, /* (77) db_options ::= */ + { 344, -3 }, /* (78) db_options ::= db_options BUFFER NK_INTEGER */ + { 344, -3 }, /* (79) db_options ::= db_options CACHEMODEL NK_STRING */ + { 344, -3 }, /* (80) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 344, -3 }, /* (81) db_options ::= db_options COMP NK_INTEGER */ + { 344, -3 }, /* (82) db_options ::= db_options DURATION NK_INTEGER */ + { 344, -3 }, /* (83) db_options ::= db_options DURATION NK_VARIABLE */ + { 344, -3 }, /* (84) db_options ::= db_options MAXROWS NK_INTEGER */ + { 344, -3 }, /* (85) db_options ::= db_options MINROWS NK_INTEGER */ + { 344, -3 }, /* (86) db_options ::= db_options KEEP integer_list */ + { 344, -3 }, /* (87) db_options ::= db_options KEEP variable_list */ + { 344, -3 }, /* (88) db_options ::= db_options PAGES NK_INTEGER */ + { 344, -3 }, /* (89) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 344, -3 }, /* (90) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 344, -3 }, /* (91) db_options ::= db_options PRECISION NK_STRING */ + { 344, -3 }, /* (92) db_options ::= db_options REPLICA NK_INTEGER */ + { 344, -3 }, /* (93) db_options ::= db_options VGROUPS NK_INTEGER */ + { 344, -3 }, /* (94) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 344, -3 }, /* (95) db_options ::= db_options RETENTIONS retention_list */ + { 344, -3 }, /* (96) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 344, -3 }, /* (97) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 344, -3 }, /* (98) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 344, -3 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 344, -4 }, /* (100) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 344, -3 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 344, -4 }, /* (102) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 344, -3 }, /* (103) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 344, -3 }, /* (104) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 344, -3 }, /* (105) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 344, -3 }, /* (106) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 344, -3 }, /* (107) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 346, -1 }, /* (108) alter_db_options ::= alter_db_option */ + { 346, -2 }, /* (109) alter_db_options ::= alter_db_options alter_db_option */ + { 351, -2 }, /* (110) alter_db_option ::= BUFFER NK_INTEGER */ + { 351, -2 }, /* (111) alter_db_option ::= CACHEMODEL NK_STRING */ + { 351, -2 }, /* (112) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 351, -2 }, /* (113) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 351, -2 }, /* (114) alter_db_option ::= KEEP integer_list */ + { 351, -2 }, /* (115) alter_db_option ::= KEEP variable_list */ + { 351, -2 }, /* (116) alter_db_option ::= PAGES NK_INTEGER */ + { 351, -2 }, /* (117) alter_db_option ::= REPLICA NK_INTEGER */ + { 351, -2 }, /* (118) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 351, -2 }, /* (119) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 348, -1 }, /* (120) integer_list ::= NK_INTEGER */ + { 348, -3 }, /* (121) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 349, -1 }, /* (122) variable_list ::= NK_VARIABLE */ + { 349, -3 }, /* (123) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 350, -1 }, /* (124) retention_list ::= retention */ + { 350, -3 }, /* (125) retention_list ::= retention_list NK_COMMA retention */ + { 352, -3 }, /* (126) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 347, 0 }, /* (127) speed_opt ::= */ + { 347, -2 }, /* (128) speed_opt ::= MAX_SPEED NK_INTEGER */ + { 328, -9 }, /* (129) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 328, -3 }, /* (130) cmd ::= CREATE TABLE multi_create_clause */ + { 328, -9 }, /* (131) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 328, -3 }, /* (132) cmd ::= DROP TABLE multi_drop_clause */ + { 328, -4 }, /* (133) cmd ::= DROP STABLE exists_opt full_table_name */ + { 328, -3 }, /* (134) cmd ::= ALTER TABLE alter_table_clause */ + { 328, -3 }, /* (135) cmd ::= ALTER STABLE alter_table_clause */ + { 360, -2 }, /* (136) alter_table_clause ::= full_table_name alter_table_options */ + { 360, -5 }, /* (137) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 360, -4 }, /* (138) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 360, -5 }, /* (139) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 360, -5 }, /* (140) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 360, -5 }, /* (141) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 360, -4 }, /* (142) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 360, -5 }, /* (143) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 360, -5 }, /* (144) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 360, -6 }, /* (145) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 357, -1 }, /* (146) multi_create_clause ::= create_subtable_clause */ + { 357, -2 }, /* (147) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 365, -10 }, /* (148) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 359, -1 }, /* (149) multi_drop_clause ::= drop_table_clause */ + { 359, -3 }, /* (150) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + { 368, -2 }, /* (151) drop_table_clause ::= exists_opt full_table_name */ + { 366, 0 }, /* (152) specific_cols_opt ::= */ + { 366, -3 }, /* (153) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 353, -1 }, /* (154) full_table_name ::= table_name */ + { 353, -3 }, /* (155) full_table_name ::= db_name NK_DOT table_name */ + { 354, -1 }, /* (156) column_def_list ::= column_def */ + { 354, -3 }, /* (157) column_def_list ::= column_def_list NK_COMMA column_def */ + { 371, -2 }, /* (158) column_def ::= column_name type_name */ + { 371, -4 }, /* (159) column_def ::= column_name type_name COMMENT NK_STRING */ + { 363, -1 }, /* (160) type_name ::= BOOL */ + { 363, -1 }, /* (161) type_name ::= TINYINT */ + { 363, -1 }, /* (162) type_name ::= SMALLINT */ + { 363, -1 }, /* (163) type_name ::= INT */ + { 363, -1 }, /* (164) type_name ::= INTEGER */ + { 363, -1 }, /* (165) type_name ::= BIGINT */ + { 363, -1 }, /* (166) type_name ::= FLOAT */ + { 363, -1 }, /* (167) type_name ::= DOUBLE */ + { 363, -4 }, /* (168) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 363, -1 }, /* (169) type_name ::= TIMESTAMP */ + { 363, -4 }, /* (170) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 363, -2 }, /* (171) type_name ::= TINYINT UNSIGNED */ + { 363, -2 }, /* (172) type_name ::= SMALLINT UNSIGNED */ + { 363, -2 }, /* (173) type_name ::= INT UNSIGNED */ + { 363, -2 }, /* (174) type_name ::= BIGINT UNSIGNED */ + { 363, -1 }, /* (175) type_name ::= JSON */ + { 363, -4 }, /* (176) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 363, -1 }, /* (177) type_name ::= MEDIUMBLOB */ + { 363, -1 }, /* (178) type_name ::= BLOB */ + { 363, -4 }, /* (179) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 363, -1 }, /* (180) type_name ::= DECIMAL */ + { 363, -4 }, /* (181) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 363, -6 }, /* (182) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 355, 0 }, /* (183) tags_def_opt ::= */ + { 355, -1 }, /* (184) tags_def_opt ::= tags_def */ + { 358, -4 }, /* (185) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 356, 0 }, /* (186) table_options ::= */ + { 356, -3 }, /* (187) table_options ::= table_options COMMENT NK_STRING */ + { 356, -3 }, /* (188) table_options ::= table_options MAX_DELAY duration_list */ + { 356, -3 }, /* (189) table_options ::= table_options WATERMARK duration_list */ + { 356, -5 }, /* (190) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 356, -3 }, /* (191) table_options ::= table_options TTL NK_INTEGER */ + { 356, -5 }, /* (192) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 356, -3 }, /* (193) table_options ::= table_options DELETE_MARK duration_list */ + { 361, -1 }, /* (194) alter_table_options ::= alter_table_option */ + { 361, -2 }, /* (195) alter_table_options ::= alter_table_options alter_table_option */ + { 374, -2 }, /* (196) alter_table_option ::= COMMENT NK_STRING */ + { 374, -2 }, /* (197) alter_table_option ::= TTL NK_INTEGER */ + { 372, -1 }, /* (198) duration_list ::= duration_literal */ + { 372, -3 }, /* (199) duration_list ::= duration_list NK_COMMA duration_literal */ + { 373, -1 }, /* (200) rollup_func_list ::= rollup_func_name */ + { 373, -3 }, /* (201) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 376, -1 }, /* (202) rollup_func_name ::= function_name */ + { 376, -1 }, /* (203) rollup_func_name ::= FIRST */ + { 376, -1 }, /* (204) rollup_func_name ::= LAST */ + { 369, -1 }, /* (205) col_name_list ::= col_name */ + { 369, -3 }, /* (206) col_name_list ::= col_name_list NK_COMMA col_name */ + { 378, -1 }, /* (207) col_name ::= column_name */ + { 328, -2 }, /* (208) cmd ::= SHOW DNODES */ + { 328, -2 }, /* (209) cmd ::= SHOW USERS */ + { 328, -3 }, /* (210) cmd ::= SHOW USER PRIVILEGES */ + { 328, -2 }, /* (211) cmd ::= SHOW DATABASES */ + { 328, -4 }, /* (212) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 328, -4 }, /* (213) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 328, -3 }, /* (214) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 328, -2 }, /* (215) cmd ::= SHOW MNODES */ + { 328, -2 }, /* (216) cmd ::= SHOW QNODES */ + { 328, -2 }, /* (217) cmd ::= SHOW FUNCTIONS */ + { 328, -5 }, /* (218) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 328, -2 }, /* (219) cmd ::= SHOW STREAMS */ + { 328, -2 }, /* (220) cmd ::= SHOW ACCOUNTS */ + { 328, -2 }, /* (221) cmd ::= SHOW APPS */ + { 328, -2 }, /* (222) cmd ::= SHOW CONNECTIONS */ + { 328, -2 }, /* (223) cmd ::= SHOW LICENCES */ + { 328, -2 }, /* (224) cmd ::= SHOW GRANTS */ + { 328, -4 }, /* (225) cmd ::= SHOW CREATE DATABASE db_name */ + { 328, -4 }, /* (226) cmd ::= SHOW CREATE TABLE full_table_name */ + { 328, -4 }, /* (227) cmd ::= SHOW CREATE STABLE full_table_name */ + { 328, -2 }, /* (228) cmd ::= SHOW QUERIES */ + { 328, -2 }, /* (229) cmd ::= SHOW SCORES */ + { 328, -2 }, /* (230) cmd ::= SHOW TOPICS */ + { 328, -2 }, /* (231) cmd ::= SHOW VARIABLES */ + { 328, -3 }, /* (232) cmd ::= SHOW CLUSTER VARIABLES */ + { 328, -3 }, /* (233) cmd ::= SHOW LOCAL VARIABLES */ + { 328, -5 }, /* (234) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 328, -2 }, /* (235) cmd ::= SHOW BNODES */ + { 328, -2 }, /* (236) cmd ::= SHOW SNODES */ + { 328, -2 }, /* (237) cmd ::= SHOW CLUSTER */ + { 328, -2 }, /* (238) cmd ::= SHOW TRANSACTIONS */ + { 328, -4 }, /* (239) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 328, -2 }, /* (240) cmd ::= SHOW CONSUMERS */ + { 328, -2 }, /* (241) cmd ::= SHOW SUBSCRIPTIONS */ + { 328, -5 }, /* (242) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 328, -7 }, /* (243) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 328, -3 }, /* (244) cmd ::= SHOW VNODES NK_INTEGER */ + { 328, -3 }, /* (245) cmd ::= SHOW VNODES NK_STRING */ + { 328, -3 }, /* (246) cmd ::= SHOW db_name_cond_opt ALIVE */ + { 328, -3 }, /* (247) cmd ::= SHOW CLUSTER ALIVE */ + { 379, 0 }, /* (248) db_name_cond_opt ::= */ + { 379, -2 }, /* (249) db_name_cond_opt ::= db_name NK_DOT */ + { 380, 0 }, /* (250) like_pattern_opt ::= */ + { 380, -2 }, /* (251) like_pattern_opt ::= LIKE NK_STRING */ + { 381, -1 }, /* (252) table_name_cond ::= table_name */ + { 382, 0 }, /* (253) from_db_opt ::= */ + { 382, -2 }, /* (254) from_db_opt ::= FROM db_name */ + { 383, 0 }, /* (255) tag_list_opt ::= */ + { 383, -1 }, /* (256) tag_list_opt ::= tag_item */ + { 383, -3 }, /* (257) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 384, -1 }, /* (258) tag_item ::= TBNAME */ + { 384, -1 }, /* (259) tag_item ::= QTAGS */ + { 384, -1 }, /* (260) tag_item ::= column_name */ + { 384, -2 }, /* (261) tag_item ::= column_name column_alias */ + { 384, -3 }, /* (262) tag_item ::= column_name AS column_alias */ + { 328, -8 }, /* (263) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + { 328, -9 }, /* (264) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + { 328, -4 }, /* (265) cmd ::= DROP INDEX exists_opt full_index_name */ + { 386, -1 }, /* (266) full_index_name ::= index_name */ + { 386, -3 }, /* (267) full_index_name ::= db_name NK_DOT index_name */ + { 387, -10 }, /* (268) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 387, -12 }, /* (269) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 389, -1 }, /* (270) func_list ::= func */ + { 389, -3 }, /* (271) func_list ::= func_list NK_COMMA func */ + { 392, -4 }, /* (272) func ::= sma_func_name NK_LP expression_list NK_RP */ + { 393, -1 }, /* (273) sma_func_name ::= function_name */ + { 393, -1 }, /* (274) sma_func_name ::= COUNT */ + { 393, -1 }, /* (275) sma_func_name ::= FIRST */ + { 393, -1 }, /* (276) sma_func_name ::= LAST */ + { 393, -1 }, /* (277) sma_func_name ::= LAST_ROW */ + { 391, 0 }, /* (278) sma_stream_opt ::= */ + { 391, -3 }, /* (279) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 391, -3 }, /* (280) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 391, -3 }, /* (281) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 328, -6 }, /* (282) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 328, -7 }, /* (283) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 328, -9 }, /* (284) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 328, -7 }, /* (285) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 328, -9 }, /* (286) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 328, -4 }, /* (287) cmd ::= DROP TOPIC exists_opt topic_name */ + { 328, -7 }, /* (288) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 328, -2 }, /* (289) cmd ::= DESC full_table_name */ + { 328, -2 }, /* (290) cmd ::= DESCRIBE full_table_name */ + { 328, -3 }, /* (291) cmd ::= RESET QUERY CACHE */ + { 328, -4 }, /* (292) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 328, -4 }, /* (293) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + { 396, 0 }, /* (294) analyze_opt ::= */ + { 396, -1 }, /* (295) analyze_opt ::= ANALYZE */ + { 397, 0 }, /* (296) explain_options ::= */ + { 397, -3 }, /* (297) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 397, -3 }, /* (298) explain_options ::= explain_options RATIO NK_FLOAT */ + { 328, -10 }, /* (299) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 328, -4 }, /* (300) cmd ::= DROP FUNCTION exists_opt function_name */ + { 399, 0 }, /* (301) agg_func_opt ::= */ + { 399, -1 }, /* (302) agg_func_opt ::= AGGREGATE */ + { 400, 0 }, /* (303) bufsize_opt ::= */ + { 400, -2 }, /* (304) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 328, -12 }, /* (305) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + { 328, -4 }, /* (306) cmd ::= DROP STREAM exists_opt stream_name */ + { 403, 0 }, /* (307) col_list_opt ::= */ + { 403, -3 }, /* (308) col_list_opt ::= NK_LP col_name_list NK_RP */ + { 404, 0 }, /* (309) tag_def_or_ref_opt ::= */ + { 404, -1 }, /* (310) tag_def_or_ref_opt ::= tags_def */ + { 404, -4 }, /* (311) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + { 402, 0 }, /* (312) stream_options ::= */ + { 402, -3 }, /* (313) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 402, -3 }, /* (314) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 402, -4 }, /* (315) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 402, -3 }, /* (316) stream_options ::= stream_options WATERMARK duration_literal */ + { 402, -4 }, /* (317) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 402, -3 }, /* (318) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 402, -3 }, /* (319) stream_options ::= stream_options DELETE_MARK duration_literal */ + { 402, -4 }, /* (320) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + { 405, 0 }, /* (321) subtable_opt ::= */ + { 405, -4 }, /* (322) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 328, -3 }, /* (323) cmd ::= KILL CONNECTION NK_INTEGER */ + { 328, -3 }, /* (324) cmd ::= KILL QUERY NK_STRING */ + { 328, -3 }, /* (325) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 328, -2 }, /* (326) cmd ::= BALANCE VGROUP */ + { 328, -4 }, /* (327) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 328, -4 }, /* (328) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 328, -3 }, /* (329) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 407, -2 }, /* (330) dnode_list ::= DNODE NK_INTEGER */ + { 407, -3 }, /* (331) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 328, -4 }, /* (332) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 328, -1 }, /* (333) cmd ::= query_or_subquery */ + { 328, -1 }, /* (334) cmd ::= insert_query */ + { 398, -7 }, /* (335) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 398, -4 }, /* (336) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + { 331, -1 }, /* (337) literal ::= NK_INTEGER */ + { 331, -1 }, /* (338) literal ::= NK_FLOAT */ + { 331, -1 }, /* (339) literal ::= NK_STRING */ + { 331, -1 }, /* (340) literal ::= NK_BOOL */ + { 331, -2 }, /* (341) literal ::= TIMESTAMP NK_STRING */ + { 331, -1 }, /* (342) literal ::= duration_literal */ + { 331, -1 }, /* (343) literal ::= NULL */ + { 331, -1 }, /* (344) literal ::= NK_QUESTION */ + { 375, -1 }, /* (345) duration_literal ::= NK_VARIABLE */ + { 409, -1 }, /* (346) signed ::= NK_INTEGER */ + { 409, -2 }, /* (347) signed ::= NK_PLUS NK_INTEGER */ + { 409, -2 }, /* (348) signed ::= NK_MINUS NK_INTEGER */ + { 409, -1 }, /* (349) signed ::= NK_FLOAT */ + { 409, -2 }, /* (350) signed ::= NK_PLUS NK_FLOAT */ + { 409, -2 }, /* (351) signed ::= NK_MINUS NK_FLOAT */ + { 364, -1 }, /* (352) signed_literal ::= signed */ + { 364, -1 }, /* (353) signed_literal ::= NK_STRING */ + { 364, -1 }, /* (354) signed_literal ::= NK_BOOL */ + { 364, -2 }, /* (355) signed_literal ::= TIMESTAMP NK_STRING */ + { 364, -1 }, /* (356) signed_literal ::= duration_literal */ + { 364, -1 }, /* (357) signed_literal ::= NULL */ + { 364, -1 }, /* (358) signed_literal ::= literal_func */ + { 364, -1 }, /* (359) signed_literal ::= NK_QUESTION */ + { 411, -1 }, /* (360) literal_list ::= signed_literal */ + { 411, -3 }, /* (361) literal_list ::= literal_list NK_COMMA signed_literal */ + { 339, -1 }, /* (362) db_name ::= NK_ID */ + { 370, -1 }, /* (363) table_name ::= NK_ID */ + { 362, -1 }, /* (364) column_name ::= NK_ID */ + { 377, -1 }, /* (365) function_name ::= NK_ID */ + { 412, -1 }, /* (366) table_alias ::= NK_ID */ + { 385, -1 }, /* (367) column_alias ::= NK_ID */ + { 333, -1 }, /* (368) user_name ::= NK_ID */ + { 340, -1 }, /* (369) topic_name ::= NK_ID */ + { 401, -1 }, /* (370) stream_name ::= NK_ID */ + { 395, -1 }, /* (371) cgroup_name ::= NK_ID */ + { 388, -1 }, /* (372) index_name ::= NK_ID */ + { 413, -1 }, /* (373) expr_or_subquery ::= expression */ + { 406, -1 }, /* (374) expression ::= literal */ + { 406, -1 }, /* (375) expression ::= pseudo_column */ + { 406, -1 }, /* (376) expression ::= column_reference */ + { 406, -1 }, /* (377) expression ::= function_expression */ + { 406, -1 }, /* (378) expression ::= case_when_expression */ + { 406, -3 }, /* (379) expression ::= NK_LP expression NK_RP */ + { 406, -2 }, /* (380) expression ::= NK_PLUS expr_or_subquery */ + { 406, -2 }, /* (381) expression ::= NK_MINUS expr_or_subquery */ + { 406, -3 }, /* (382) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 406, -3 }, /* (383) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 406, -3 }, /* (384) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 406, -3 }, /* (385) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 406, -3 }, /* (386) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 406, -3 }, /* (387) expression ::= column_reference NK_ARROW NK_STRING */ + { 406, -3 }, /* (388) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 406, -3 }, /* (389) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 367, -1 }, /* (390) expression_list ::= expr_or_subquery */ + { 367, -3 }, /* (391) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 415, -1 }, /* (392) column_reference ::= column_name */ + { 415, -3 }, /* (393) column_reference ::= table_name NK_DOT column_name */ + { 414, -1 }, /* (394) pseudo_column ::= ROWTS */ + { 414, -1 }, /* (395) pseudo_column ::= TBNAME */ + { 414, -3 }, /* (396) pseudo_column ::= table_name NK_DOT TBNAME */ + { 414, -1 }, /* (397) pseudo_column ::= QSTART */ + { 414, -1 }, /* (398) pseudo_column ::= QEND */ + { 414, -1 }, /* (399) pseudo_column ::= QDURATION */ + { 414, -1 }, /* (400) pseudo_column ::= WSTART */ + { 414, -1 }, /* (401) pseudo_column ::= WEND */ + { 414, -1 }, /* (402) pseudo_column ::= WDURATION */ + { 414, -1 }, /* (403) pseudo_column ::= IROWTS */ + { 414, -1 }, /* (404) pseudo_column ::= ISFILLED */ + { 414, -1 }, /* (405) pseudo_column ::= QTAGS */ + { 416, -4 }, /* (406) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 416, -4 }, /* (407) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 416, -6 }, /* (408) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 416, -1 }, /* (409) function_expression ::= literal_func */ + { 410, -3 }, /* (410) literal_func ::= noarg_func NK_LP NK_RP */ + { 410, -1 }, /* (411) literal_func ::= NOW */ + { 420, -1 }, /* (412) noarg_func ::= NOW */ + { 420, -1 }, /* (413) noarg_func ::= TODAY */ + { 420, -1 }, /* (414) noarg_func ::= TIMEZONE */ + { 420, -1 }, /* (415) noarg_func ::= DATABASE */ + { 420, -1 }, /* (416) noarg_func ::= CLIENT_VERSION */ + { 420, -1 }, /* (417) noarg_func ::= SERVER_VERSION */ + { 420, -1 }, /* (418) noarg_func ::= SERVER_STATUS */ + { 420, -1 }, /* (419) noarg_func ::= CURRENT_USER */ + { 420, -1 }, /* (420) noarg_func ::= USER */ + { 418, -1 }, /* (421) star_func ::= COUNT */ + { 418, -1 }, /* (422) star_func ::= FIRST */ + { 418, -1 }, /* (423) star_func ::= LAST */ + { 418, -1 }, /* (424) star_func ::= LAST_ROW */ + { 419, -1 }, /* (425) star_func_para_list ::= NK_STAR */ + { 419, -1 }, /* (426) star_func_para_list ::= other_para_list */ + { 421, -1 }, /* (427) other_para_list ::= star_func_para */ + { 421, -3 }, /* (428) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 422, -1 }, /* (429) star_func_para ::= expr_or_subquery */ + { 422, -3 }, /* (430) star_func_para ::= table_name NK_DOT NK_STAR */ + { 417, -4 }, /* (431) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 417, -5 }, /* (432) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 423, -1 }, /* (433) when_then_list ::= when_then_expr */ + { 423, -2 }, /* (434) when_then_list ::= when_then_list when_then_expr */ + { 426, -4 }, /* (435) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 424, 0 }, /* (436) case_when_else_opt ::= */ + { 424, -2 }, /* (437) case_when_else_opt ::= ELSE common_expression */ + { 427, -3 }, /* (438) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 427, -5 }, /* (439) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 427, -6 }, /* (440) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 427, -3 }, /* (441) predicate ::= expr_or_subquery IS NULL */ + { 427, -4 }, /* (442) predicate ::= expr_or_subquery IS NOT NULL */ + { 427, -3 }, /* (443) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 428, -1 }, /* (444) compare_op ::= NK_LT */ + { 428, -1 }, /* (445) compare_op ::= NK_GT */ + { 428, -1 }, /* (446) compare_op ::= NK_LE */ + { 428, -1 }, /* (447) compare_op ::= NK_GE */ + { 428, -1 }, /* (448) compare_op ::= NK_NE */ + { 428, -1 }, /* (449) compare_op ::= NK_EQ */ + { 428, -1 }, /* (450) compare_op ::= LIKE */ + { 428, -2 }, /* (451) compare_op ::= NOT LIKE */ + { 428, -1 }, /* (452) compare_op ::= MATCH */ + { 428, -1 }, /* (453) compare_op ::= NMATCH */ + { 428, -1 }, /* (454) compare_op ::= CONTAINS */ + { 429, -1 }, /* (455) in_op ::= IN */ + { 429, -2 }, /* (456) in_op ::= NOT IN */ + { 430, -3 }, /* (457) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 431, -1 }, /* (458) boolean_value_expression ::= boolean_primary */ + { 431, -2 }, /* (459) boolean_value_expression ::= NOT boolean_primary */ + { 431, -3 }, /* (460) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 431, -3 }, /* (461) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 432, -1 }, /* (462) boolean_primary ::= predicate */ + { 432, -3 }, /* (463) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 425, -1 }, /* (464) common_expression ::= expr_or_subquery */ + { 425, -1 }, /* (465) common_expression ::= boolean_value_expression */ + { 433, 0 }, /* (466) from_clause_opt ::= */ + { 433, -2 }, /* (467) from_clause_opt ::= FROM table_reference_list */ + { 434, -1 }, /* (468) table_reference_list ::= table_reference */ + { 434, -3 }, /* (469) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 435, -1 }, /* (470) table_reference ::= table_primary */ + { 435, -1 }, /* (471) table_reference ::= joined_table */ + { 436, -2 }, /* (472) table_primary ::= table_name alias_opt */ + { 436, -4 }, /* (473) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 436, -2 }, /* (474) table_primary ::= subquery alias_opt */ + { 436, -1 }, /* (475) table_primary ::= parenthesized_joined_table */ + { 438, 0 }, /* (476) alias_opt ::= */ + { 438, -1 }, /* (477) alias_opt ::= table_alias */ + { 438, -2 }, /* (478) alias_opt ::= AS table_alias */ + { 440, -3 }, /* (479) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 440, -3 }, /* (480) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 437, -6 }, /* (481) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 441, 0 }, /* (482) join_type ::= */ + { 441, -1 }, /* (483) join_type ::= INNER */ + { 443, -12 }, /* (484) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 444, 0 }, /* (485) set_quantifier_opt ::= */ + { 444, -1 }, /* (486) set_quantifier_opt ::= DISTINCT */ + { 444, -1 }, /* (487) set_quantifier_opt ::= ALL */ + { 445, -1 }, /* (488) select_list ::= select_item */ + { 445, -3 }, /* (489) select_list ::= select_list NK_COMMA select_item */ + { 453, -1 }, /* (490) select_item ::= NK_STAR */ + { 453, -1 }, /* (491) select_item ::= common_expression */ + { 453, -2 }, /* (492) select_item ::= common_expression column_alias */ + { 453, -3 }, /* (493) select_item ::= common_expression AS column_alias */ + { 453, -3 }, /* (494) select_item ::= table_name NK_DOT NK_STAR */ + { 408, 0 }, /* (495) where_clause_opt ::= */ + { 408, -2 }, /* (496) where_clause_opt ::= WHERE search_condition */ + { 446, 0 }, /* (497) partition_by_clause_opt ::= */ + { 446, -3 }, /* (498) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 454, -1 }, /* (499) partition_list ::= partition_item */ + { 454, -3 }, /* (500) partition_list ::= partition_list NK_COMMA partition_item */ + { 455, -1 }, /* (501) partition_item ::= expr_or_subquery */ + { 455, -2 }, /* (502) partition_item ::= expr_or_subquery column_alias */ + { 455, -3 }, /* (503) partition_item ::= expr_or_subquery AS column_alias */ + { 450, 0 }, /* (504) twindow_clause_opt ::= */ + { 450, -6 }, /* (505) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 450, -4 }, /* (506) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 450, -6 }, /* (507) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 450, -8 }, /* (508) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 450, -7 }, /* (509) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 390, 0 }, /* (510) sliding_opt ::= */ + { 390, -4 }, /* (511) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 449, 0 }, /* (512) fill_opt ::= */ + { 449, -4 }, /* (513) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 449, -6 }, /* (514) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 449, -6 }, /* (515) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + { 456, -1 }, /* (516) fill_mode ::= NONE */ + { 456, -1 }, /* (517) fill_mode ::= PREV */ + { 456, -1 }, /* (518) fill_mode ::= NULL */ + { 456, -1 }, /* (519) fill_mode ::= NULL_F */ + { 456, -1 }, /* (520) fill_mode ::= LINEAR */ + { 456, -1 }, /* (521) fill_mode ::= NEXT */ + { 451, 0 }, /* (522) group_by_clause_opt ::= */ + { 451, -3 }, /* (523) group_by_clause_opt ::= GROUP BY group_by_list */ + { 457, -1 }, /* (524) group_by_list ::= expr_or_subquery */ + { 457, -3 }, /* (525) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 452, 0 }, /* (526) having_clause_opt ::= */ + { 452, -2 }, /* (527) having_clause_opt ::= HAVING search_condition */ + { 447, 0 }, /* (528) range_opt ::= */ + { 447, -6 }, /* (529) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 448, 0 }, /* (530) every_opt ::= */ + { 448, -4 }, /* (531) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 458, -4 }, /* (532) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 459, -1 }, /* (533) query_simple ::= query_specification */ + { 459, -1 }, /* (534) query_simple ::= union_query_expression */ + { 463, -4 }, /* (535) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 463, -3 }, /* (536) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 464, -1 }, /* (537) query_simple_or_subquery ::= query_simple */ + { 464, -1 }, /* (538) query_simple_or_subquery ::= subquery */ + { 394, -1 }, /* (539) query_or_subquery ::= query_expression */ + { 394, -1 }, /* (540) query_or_subquery ::= subquery */ + { 460, 0 }, /* (541) order_by_clause_opt ::= */ + { 460, -3 }, /* (542) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 461, 0 }, /* (543) slimit_clause_opt ::= */ + { 461, -2 }, /* (544) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 461, -4 }, /* (545) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 461, -4 }, /* (546) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 462, 0 }, /* (547) limit_clause_opt ::= */ + { 462, -2 }, /* (548) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 462, -4 }, /* (549) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 462, -4 }, /* (550) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 439, -3 }, /* (551) subquery ::= NK_LP query_expression NK_RP */ + { 439, -3 }, /* (552) subquery ::= NK_LP subquery NK_RP */ + { 442, -1 }, /* (553) search_condition ::= common_expression */ + { 465, -1 }, /* (554) sort_specification_list ::= sort_specification */ + { 465, -3 }, /* (555) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 466, -3 }, /* (556) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 467, 0 }, /* (557) ordering_specification_opt ::= */ + { 467, -1 }, /* (558) ordering_specification_opt ::= ASC */ + { 467, -1 }, /* (559) ordering_specification_opt ::= DESC */ + { 468, 0 }, /* (560) null_ordering_opt ::= */ + { 468, -2 }, /* (561) null_ordering_opt ::= NULLS FIRST */ + { 468, -2 }, /* (562) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3643,11 +3704,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,325,&yymsp[0].minor); + yy_destructor(yypParser,329,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,326,&yymsp[0].minor); + yy_destructor(yypParser,330,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3661,20 +3722,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,325,&yymsp[-2].minor); +{ yy_destructor(yypParser,329,&yymsp[-2].minor); { } - yy_destructor(yypParser,327,&yymsp[0].minor); + yy_destructor(yypParser,331,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,328,&yymsp[0].minor); +{ yy_destructor(yypParser,332,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,326,&yymsp[-1].minor); +{ yy_destructor(yypParser,330,&yymsp[-1].minor); { } - yy_destructor(yypParser,328,&yymsp[0].minor); + yy_destructor(yypParser,332,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3688,80 +3749,81 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,327,&yymsp[0].minor); + yy_destructor(yypParser,331,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy0, yymsp[0].minor.yy287); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy225, &yymsp[-1].minor.yy0, yymsp[0].minor.yy705); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy225, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy225, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy77, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy225, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy225); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy287 = 1; } +{ yymsp[1].minor.yy705 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy287 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy705 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy641, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy717, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy641, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy641 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy717 = yymsp[0].minor.yy717; } - yymsp[0].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy641 = yymsp[0].minor.yy641; } + yymsp[0].minor.yy641 = yylhsminor.yy641; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy641 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy717 = yymsp[-2].minor.yy717 | yymsp[0].minor.yy717; } - yymsp[-2].minor.yy717 = yylhsminor.yy717; +{ yylhsminor.yy641 = yymsp[-2].minor.yy641 | yymsp[0].minor.yy641; } + yymsp[-2].minor.yy641 = yylhsminor.yy641; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy641 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy717 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy641 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy225 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy225 = yylhsminor.yy225; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy77 = yymsp[-2].minor.yy77; } - yymsp[-2].minor.yy77 = yylhsminor.yy77; +{ yylhsminor.yy225 = yymsp[-2].minor.yy225; } + yymsp[-2].minor.yy225 = yylhsminor.yy225; break; case 42: /* priv_level ::= topic_name */ - case 460: /* alias_opt ::= table_alias */ yytestcase(yyruleno==460); -{ yylhsminor.yy77 = yymsp[0].minor.yy77; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 273: /* sma_func_name ::= function_name */ yytestcase(yyruleno==273); + case 477: /* alias_opt ::= table_alias */ yytestcase(yyruleno==477); +{ yylhsminor.yy225 = yymsp[0].minor.yy225; } + yymsp[0].minor.yy225 = yylhsminor.yy225; break; case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy77, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy225, NULL); } break; case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy0); } break; case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy103); } break; case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy841); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy225, yymsp[0].minor.yy103); } break; case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3778,46 +3840,50 @@ static YYACTIONTYPE yy_reduce( case 51: /* dnode_endpoint ::= NK_STRING */ case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 346: /* db_name ::= NK_ID */ yytestcase(yyruleno==346); - case 347: /* table_name ::= NK_ID */ yytestcase(yyruleno==347); - case 348: /* column_name ::= NK_ID */ yytestcase(yyruleno==348); - case 349: /* function_name ::= NK_ID */ yytestcase(yyruleno==349); - case 350: /* table_alias ::= NK_ID */ yytestcase(yyruleno==350); - case 351: /* column_alias ::= NK_ID */ yytestcase(yyruleno==351); - case 352: /* user_name ::= NK_ID */ yytestcase(yyruleno==352); - case 353: /* topic_name ::= NK_ID */ yytestcase(yyruleno==353); - case 354: /* stream_name ::= NK_ID */ yytestcase(yyruleno==354); - case 355: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==355); - case 356: /* index_name ::= NK_ID */ yytestcase(yyruleno==356); - case 395: /* noarg_func ::= NOW */ yytestcase(yyruleno==395); - case 396: /* noarg_func ::= TODAY */ yytestcase(yyruleno==396); - case 397: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==397); - case 398: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==398); - case 399: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==399); - case 400: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==400); - case 401: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==401); - case 402: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==402); - case 403: /* noarg_func ::= USER */ yytestcase(yyruleno==403); - case 404: /* star_func ::= COUNT */ yytestcase(yyruleno==404); - case 405: /* star_func ::= FIRST */ yytestcase(yyruleno==405); - case 406: /* star_func ::= LAST */ yytestcase(yyruleno==406); - case 407: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==407); -{ yylhsminor.yy77 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy77 = yylhsminor.yy77; + case 274: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==274); + case 275: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==275); + case 276: /* sma_func_name ::= LAST */ yytestcase(yyruleno==276); + case 277: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==277); + case 362: /* db_name ::= NK_ID */ yytestcase(yyruleno==362); + case 363: /* table_name ::= NK_ID */ yytestcase(yyruleno==363); + case 364: /* column_name ::= NK_ID */ yytestcase(yyruleno==364); + case 365: /* function_name ::= NK_ID */ yytestcase(yyruleno==365); + case 366: /* table_alias ::= NK_ID */ yytestcase(yyruleno==366); + case 367: /* column_alias ::= NK_ID */ yytestcase(yyruleno==367); + case 368: /* user_name ::= NK_ID */ yytestcase(yyruleno==368); + case 369: /* topic_name ::= NK_ID */ yytestcase(yyruleno==369); + case 370: /* stream_name ::= NK_ID */ yytestcase(yyruleno==370); + case 371: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==371); + case 372: /* index_name ::= NK_ID */ yytestcase(yyruleno==372); + case 412: /* noarg_func ::= NOW */ yytestcase(yyruleno==412); + case 413: /* noarg_func ::= TODAY */ yytestcase(yyruleno==413); + case 414: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==414); + case 415: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==415); + case 416: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==416); + case 417: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==417); + case 418: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==418); + case 419: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==419); + case 420: /* noarg_func ::= USER */ yytestcase(yyruleno==420); + case 421: /* star_func ::= COUNT */ yytestcase(yyruleno==421); + case 422: /* star_func ::= FIRST */ yytestcase(yyruleno==422); + case 423: /* star_func ::= LAST */ yytestcase(yyruleno==423); + case 424: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==424); +{ yylhsminor.yy225 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy225 = yylhsminor.yy225; break; case 54: /* force_opt ::= */ - case 73: /* not_exists_opt ::= */ yytestcase(yyruleno==73); - case 75: /* exists_opt ::= */ yytestcase(yyruleno==75); - case 284: /* analyze_opt ::= */ yytestcase(yyruleno==284); - case 291: /* agg_func_opt ::= */ yytestcase(yyruleno==291); - case 468: /* set_quantifier_opt ::= */ yytestcase(yyruleno==468); -{ yymsp[1].minor.yy841 = false; } + case 74: /* not_exists_opt ::= */ yytestcase(yyruleno==74); + case 76: /* exists_opt ::= */ yytestcase(yyruleno==76); + case 294: /* analyze_opt ::= */ yytestcase(yyruleno==294); + case 301: /* agg_func_opt ::= */ yytestcase(yyruleno==301); + case 485: /* set_quantifier_opt ::= */ yytestcase(yyruleno==485); +{ yymsp[1].minor.yy103 = false; } break; case 55: /* force_opt ::= FORCE */ - case 285: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==285); - case 292: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==292); - case 469: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==469); -{ yymsp[0].minor.yy841 = true; } + case 295: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==295); + case 302: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==302); + case 486: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==486); +{ yymsp[0].minor.yy103 = true; } break; case 56: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3850,1354 +3916,1377 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy841, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy103, &yymsp[-1].minor.yy225, yymsp[0].minor.yy42); } break; case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy103, &yymsp[0].minor.yy225); } break; case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy225); } break; case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy600); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy225, yymsp[0].minor.yy42); } break; case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy225); } break; case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy248); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy225, yymsp[0].minor.yy508); } + break; + case 72: /* cmd ::= COMPACT DATABASE db_name */ +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[0].minor.yy225); } break; - case 72: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy841 = true; } + case 73: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy103 = true; } break; - case 74: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy841 = true; } + case 75: /* exists_opt ::= IF EXISTS */ +{ yymsp[-1].minor.yy103 = true; } break; - case 76: /* db_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultDatabaseOptions(pCxt); } + case 77: /* db_options ::= */ +{ yymsp[1].minor.yy42 = createDefaultDatabaseOptions(pCxt); } break; - case 77: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 78: /* db_options ::= db_options BUFFER NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 78: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 79: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 79: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 80: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 80: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 81: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 81: /* db_options ::= db_options DURATION NK_INTEGER */ - case 82: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==82); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 82: /* db_options ::= db_options DURATION NK_INTEGER */ + case 83: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==83); +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 83: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 84: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 84: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 85: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 85: /* db_options ::= db_options KEEP integer_list */ - case 86: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==86); -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_KEEP, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 86: /* db_options ::= db_options KEEP integer_list */ + case 87: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==87); +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_KEEP, yymsp[0].minor.yy110); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 87: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 88: /* db_options ::= db_options PAGES NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 88: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 89: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 89: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 90: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 90: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 91: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 91: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 92: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 92: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 93: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 93: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 94: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 94: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_RETENTIONS, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 95: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_RETENTIONS, yymsp[0].minor.yy110); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 95: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 96: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 96: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 97: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 97: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 98: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 98: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 100: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-3].minor.yy42, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 100: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 102: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-3].minor.yy600, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-3].minor.yy42, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 102: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 103: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 104: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 105: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 106: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy600 = setDatabaseOption(pCxt, yymsp[-2].minor.yy600, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 107: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy600 = createAlterDatabaseOptions(pCxt); yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yylhsminor.yy600, &yymsp[0].minor.yy661); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 108: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy600 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy600, &yymsp[0].minor.yy661); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 109: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 110: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 111: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 112: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 113: /* alter_db_option ::= KEEP integer_list */ - case 114: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==114); -{ yymsp[-1].minor.yy661.type = DB_OPTION_KEEP; yymsp[-1].minor.yy661.pList = yymsp[0].minor.yy601; } - break; - case 115: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_PAGES; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 116: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 117: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_WAL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 118: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } - break; - case 119: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 120: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 316: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==316); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; - break; - case 121: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy601 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 122: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; - break; - case 123: /* retention_list ::= retention */ - case 145: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==145); - case 148: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==148); - case 155: /* column_def_list ::= column_def */ yytestcase(yyruleno==155); - case 199: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==199); - case 204: /* col_name_list ::= col_name */ yytestcase(yyruleno==204); - case 253: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==253); - case 266: /* func_list ::= func */ yytestcase(yyruleno==266); - case 344: /* literal_list ::= signed_literal */ yytestcase(yyruleno==344); - case 410: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==410); - case 416: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==416); - case 471: /* select_list ::= select_item */ yytestcase(yyruleno==471); - case 482: /* partition_list ::= partition_item */ yytestcase(yyruleno==482); - case 536: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==536); -{ yylhsminor.yy601 = createNodeList(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 124: /* retention_list ::= retention_list NK_COMMA retention */ - case 156: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==156); - case 200: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==200); - case 205: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==205); - case 254: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==254); - case 267: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==267); - case 345: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==345); - case 411: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==411); - case 472: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==472); - case 483: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==483); - case 537: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==537); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; - break; - case 125: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy600 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 126: /* speed_opt ::= */ - case 293: /* bufsize_opt ::= */ yytestcase(yyruleno==293); -{ yymsp[1].minor.yy248 = 0; } - break; - case 127: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 294: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==294); -{ yymsp[-1].minor.yy248 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } - break; - case 128: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 130: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==130); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-5].minor.yy600, yymsp[-3].minor.yy601, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } - break; - case 129: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy601); } - break; - case 131: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy601); } - break; - case 132: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } - break; - case 133: /* cmd ::= ALTER TABLE alter_table_clause */ - case 318: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==318); -{ pCxt->pRootNode = yymsp[0].minor.yy600; } - break; - case 134: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy600); } - break; - case 135: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy600 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 136: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 137: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 138: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 139: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 140: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 141: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy600 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy600, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 142: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy600 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 143: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy600 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy600, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 144: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy600 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy600, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; - break; - case 146: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 149: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==149); - case 417: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==417); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-1].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy601 = yylhsminor.yy601; + yymsp[-3].minor.yy42 = yylhsminor.yy42; + break; + case 103: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 104: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 105: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 106: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 107: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ +{ yylhsminor.yy42 = setDatabaseOption(pCxt, yymsp[-2].minor.yy42, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 108: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy42 = createAlterDatabaseOptions(pCxt); yylhsminor.yy42 = setAlterDatabaseOption(pCxt, yylhsminor.yy42, &yymsp[0].minor.yy459); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 109: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy42 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy42, &yymsp[0].minor.yy459); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; + break; + case 110: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 111: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 112: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 113: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 114: /* alter_db_option ::= KEEP integer_list */ + case 115: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==115); +{ yymsp[-1].minor.yy459.type = DB_OPTION_KEEP; yymsp[-1].minor.yy459.pList = yymsp[0].minor.yy110; } + break; + case 116: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_PAGES; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 117: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 118: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_WAL; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 119: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } + break; + case 120: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy110 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy110 = yylhsminor.yy110; + break; + case 121: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 331: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==331); +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-2].minor.yy110, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy110 = yylhsminor.yy110; + break; + case 122: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy110 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy110 = yylhsminor.yy110; + break; + case 123: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-2].minor.yy110, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy110 = yylhsminor.yy110; + break; + case 124: /* retention_list ::= retention */ + case 146: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==146); + case 149: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==149); + case 156: /* column_def_list ::= column_def */ yytestcase(yyruleno==156); + case 200: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==200); + case 205: /* col_name_list ::= col_name */ yytestcase(yyruleno==205); + case 256: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==256); + case 270: /* func_list ::= func */ yytestcase(yyruleno==270); + case 360: /* literal_list ::= signed_literal */ yytestcase(yyruleno==360); + case 427: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==427); + case 433: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==433); + case 488: /* select_list ::= select_item */ yytestcase(yyruleno==488); + case 499: /* partition_list ::= partition_item */ yytestcase(yyruleno==499); + case 554: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==554); +{ yylhsminor.yy110 = createNodeList(pCxt, yymsp[0].minor.yy42); } + yymsp[0].minor.yy110 = yylhsminor.yy110; + break; + case 125: /* retention_list ::= retention_list NK_COMMA retention */ + case 150: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==150); + case 157: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==157); + case 201: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==201); + case 206: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==206); + case 257: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==257); + case 271: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==271); + case 361: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==361); + case 428: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==428); + case 489: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==489); + case 500: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==500); + case 555: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==555); +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-2].minor.yy110, yymsp[0].minor.yy42); } + yymsp[-2].minor.yy110 = yylhsminor.yy110; + break; + case 126: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy42 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 127: /* speed_opt ::= */ + case 303: /* bufsize_opt ::= */ yytestcase(yyruleno==303); +{ yymsp[1].minor.yy508 = 0; } + break; + case 128: /* speed_opt ::= MAX_SPEED NK_INTEGER */ + case 304: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==304); +{ yymsp[-1].minor.yy508 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + break; + case 129: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 131: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==131); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy103, yymsp[-5].minor.yy42, yymsp[-3].minor.yy110, yymsp[-1].minor.yy110, yymsp[0].minor.yy42); } + break; + case 130: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy110); } + break; + case 132: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy110); } + break; + case 133: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy103, yymsp[0].minor.yy42); } + break; + case 134: /* cmd ::= ALTER TABLE alter_table_clause */ + case 333: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==333); + case 334: /* cmd ::= insert_query */ yytestcase(yyruleno==334); +{ pCxt->pRootNode = yymsp[0].minor.yy42; } + break; + case 135: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy42); } + break; + case 136: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy42 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; + break; + case 137: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy42 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy225, yymsp[0].minor.yy448); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 138: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy42 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy42, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy225); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; + break; + case 139: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy42 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy225, yymsp[0].minor.yy448); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 140: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy42 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy225, &yymsp[0].minor.yy225); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 141: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy42 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy225, yymsp[0].minor.yy448); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 142: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy42 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy42, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy225); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; + break; + case 143: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy42 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy225, yymsp[0].minor.yy448); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 144: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy42 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy42, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy225, &yymsp[0].minor.yy225); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 145: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy42 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy42, &yymsp[-2].minor.yy225, yymsp[0].minor.yy42); } + yymsp[-5].minor.yy42 = yylhsminor.yy42; + break; + case 147: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 434: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==434); +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-1].minor.yy110, yymsp[0].minor.yy42); } + yymsp[-1].minor.yy110 = yylhsminor.yy110; break; - case 147: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy600 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy841, yymsp[-8].minor.yy600, yymsp[-6].minor.yy600, yymsp[-5].minor.yy601, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - yymsp[-9].minor.yy600 = yylhsminor.yy600; + case 148: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy42 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy103, yymsp[-8].minor.yy42, yymsp[-6].minor.yy42, yymsp[-5].minor.yy110, yymsp[-2].minor.yy110, yymsp[0].minor.yy42); } + yymsp[-9].minor.yy42 = yylhsminor.yy42; break; - case 150: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy600 = createDropTableClause(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 151: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy42 = createDropTableClause(pCxt, yymsp[-1].minor.yy103, yymsp[0].minor.yy42); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 151: /* specific_cols_opt ::= */ - case 182: /* tags_def_opt ::= */ yytestcase(yyruleno==182); - case 252: /* tag_list_opt ::= */ yytestcase(yyruleno==252); - case 480: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==480); - case 504: /* group_by_clause_opt ::= */ yytestcase(yyruleno==504); - case 523: /* order_by_clause_opt ::= */ yytestcase(yyruleno==523); -{ yymsp[1].minor.yy601 = NULL; } + case 152: /* specific_cols_opt ::= */ + case 183: /* tags_def_opt ::= */ yytestcase(yyruleno==183); + case 255: /* tag_list_opt ::= */ yytestcase(yyruleno==255); + case 307: /* col_list_opt ::= */ yytestcase(yyruleno==307); + case 309: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==309); + case 497: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==497); + case 522: /* group_by_clause_opt ::= */ yytestcase(yyruleno==522); + case 541: /* order_by_clause_opt ::= */ yytestcase(yyruleno==541); +{ yymsp[1].minor.yy110 = NULL; } break; - case 152: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy601 = yymsp[-1].minor.yy601; } + case 153: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ + case 308: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==308); +{ yymsp[-2].minor.yy110 = yymsp[-1].minor.yy110; } break; - case 153: /* full_table_name ::= table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 154: /* full_table_name ::= table_name */ +{ yylhsminor.yy42 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy225, NULL); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 154: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 155: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy42 = createRealTableNode(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225, NULL); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 157: /* column_def ::= column_name type_name */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy77, yymsp[0].minor.yy888, NULL); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 158: /* column_def ::= column_name type_name */ +{ yylhsminor.yy42 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy225, yymsp[0].minor.yy448, NULL); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 158: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy600 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-2].minor.yy888, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 159: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy42 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy225, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 159: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 160: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 160: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 161: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 161: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 162: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 162: /* type_name ::= INT */ - case 163: /* type_name ::= INTEGER */ yytestcase(yyruleno==163); -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_INT); } + case 163: /* type_name ::= INT */ + case 164: /* type_name ::= INTEGER */ yytestcase(yyruleno==164); +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 164: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 165: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 165: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 166: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 166: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 167: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 167: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 168: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 168: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 169: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 169: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 170: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 170: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 171: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 171: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 172: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 172: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UINT); } + case 173: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 173: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy888 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 174: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 174: /* type_name ::= JSON */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_JSON); } + case 175: /* type_name ::= JSON */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 175: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 176: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 176: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 177: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 177: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 178: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 178: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 179: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 179: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 180: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 180: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 181: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy888 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 182: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 183: /* tags_def_opt ::= tags_def */ - case 409: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==409); -{ yylhsminor.yy601 = yymsp[0].minor.yy601; } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 184: /* tags_def_opt ::= tags_def */ + case 310: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==310); + case 426: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==426); +{ yylhsminor.yy110 = yymsp[0].minor.yy110; } + yymsp[0].minor.yy110 = yylhsminor.yy110; break; - case 184: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy601 = yymsp[-1].minor.yy601; } + case 185: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ + case 311: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==311); +{ yymsp[-3].minor.yy110 = yymsp[-1].minor.yy110; } break; - case 185: /* table_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultTableOptions(pCxt); } + case 186: /* table_options ::= */ +{ yymsp[1].minor.yy42 = createDefaultTableOptions(pCxt); } break; - case 186: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 187: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-2].minor.yy42, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 187: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 188: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-2].minor.yy42, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy110); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 188: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 189: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-2].minor.yy42, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy110); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 189: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + case 190: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-4].minor.yy42, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy110); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; break; - case 190: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 191: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-2].minor.yy42, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 191: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-4].minor.yy600, TABLE_OPTION_SMA, yymsp[-1].minor.yy601); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + case 192: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-4].minor.yy42, TABLE_OPTION_SMA, yymsp[-1].minor.yy110); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; break; - case 192: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-2].minor.yy600, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy601); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 193: /* table_options ::= table_options DELETE_MARK duration_list */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-2].minor.yy42, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy110); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 193: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy600 = createAlterTableOptions(pCxt); yylhsminor.yy600 = setTableOption(pCxt, yylhsminor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 194: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy42 = createAlterTableOptions(pCxt); yylhsminor.yy42 = setTableOption(pCxt, yylhsminor.yy42, yymsp[0].minor.yy459.type, &yymsp[0].minor.yy459.val); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 194: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy600 = setTableOption(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy661.type, &yymsp[0].minor.yy661.val); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 195: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy42 = setTableOption(pCxt, yymsp[-1].minor.yy42, yymsp[0].minor.yy459.type, &yymsp[0].minor.yy459.val); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 195: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } + case 196: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy459.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } break; - case 196: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy661.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy661.val = yymsp[0].minor.yy0; } + case 197: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy459.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy459.val = yymsp[0].minor.yy0; } break; - case 197: /* duration_list ::= duration_literal */ - case 374: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==374); -{ yylhsminor.yy601 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 198: /* duration_list ::= duration_literal */ + case 390: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==390); +{ yylhsminor.yy110 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy42)); } + yymsp[0].minor.yy110 = yylhsminor.yy110; break; - case 198: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 375: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==375); -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 199: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 391: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==391); +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-2].minor.yy110, releaseRawExprNode(pCxt, yymsp[0].minor.yy42)); } + yymsp[-2].minor.yy110 = yylhsminor.yy110; break; - case 201: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy77, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 202: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy42 = createFunctionNode(pCxt, &yymsp[0].minor.yy225, NULL); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 202: /* rollup_func_name ::= FIRST */ - case 203: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==203); - case 256: /* tag_item ::= QTAGS */ yytestcase(yyruleno==256); -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 203: /* rollup_func_name ::= FIRST */ + case 204: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==204); + case 259: /* tag_item ::= QTAGS */ yytestcase(yyruleno==259); +{ yylhsminor.yy42 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 206: /* col_name ::= column_name */ - case 257: /* tag_item ::= column_name */ yytestcase(yyruleno==257); -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 207: /* col_name ::= column_name */ + case 260: /* tag_item ::= column_name */ yytestcase(yyruleno==260); +{ yylhsminor.yy42 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy225); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 207: /* cmd ::= SHOW DNODES */ + case 208: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 208: /* cmd ::= SHOW USERS */ + case 209: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 209: /* cmd ::= SHOW USER PRIVILEGES */ + case 210: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } break; - case 210: /* cmd ::= SHOW DATABASES */ + case 211: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 211: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } + case 212: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy42, yymsp[0].minor.yy42, OP_TYPE_LIKE); } break; - case 212: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, OP_TYPE_LIKE); } + case 213: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy42, yymsp[0].minor.yy42, OP_TYPE_LIKE); } break; - case 213: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy600, NULL, OP_TYPE_LIKE); } + case 214: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy42, NULL, OP_TYPE_LIKE); } break; - case 214: /* cmd ::= SHOW MNODES */ + case 215: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 215: /* cmd ::= SHOW QNODES */ + case 216: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 216: /* cmd ::= SHOW FUNCTIONS */ + case 217: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 217: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } + case 218: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy42, yymsp[-1].minor.yy42, OP_TYPE_EQUAL); } break; - case 218: /* cmd ::= SHOW STREAMS */ + case 219: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 219: /* cmd ::= SHOW ACCOUNTS */ + case 220: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 220: /* cmd ::= SHOW APPS */ + case 221: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 221: /* cmd ::= SHOW CONNECTIONS */ + case 222: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 222: /* cmd ::= SHOW LICENCES */ - case 223: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==223); + case 223: /* cmd ::= SHOW LICENCES */ + case 224: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==224); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 224: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy77); } + case 225: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy225); } break; - case 225: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy600); } + case 226: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy42); } break; - case 226: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy600); } + case 227: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy42); } break; - case 227: /* cmd ::= SHOW QUERIES */ + case 228: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 228: /* cmd ::= SHOW SCORES */ + case 229: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 229: /* cmd ::= SHOW TOPICS */ + case 230: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 230: /* cmd ::= SHOW VARIABLES */ - case 231: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==231); + case 231: /* cmd ::= SHOW VARIABLES */ + case 232: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==232); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 232: /* cmd ::= SHOW LOCAL VARIABLES */ + case 233: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 233: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy600); } + case 234: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy42); } break; - case 234: /* cmd ::= SHOW BNODES */ + case 235: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 235: /* cmd ::= SHOW SNODES */ + case 236: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 236: /* cmd ::= SHOW CLUSTER */ + case 237: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 237: /* cmd ::= SHOW TRANSACTIONS */ + case 238: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 238: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy600); } + case 239: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy42); } break; - case 239: /* cmd ::= SHOW CONSUMERS */ + case 240: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 240: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 241: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 241: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy600, yymsp[-1].minor.yy600, OP_TYPE_EQUAL); } + case 242: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy42, yymsp[-1].minor.yy42, OP_TYPE_EQUAL); } break; - case 242: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600, yymsp[-3].minor.yy601); } + case 243: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy42, yymsp[0].minor.yy42, yymsp[-3].minor.yy110); } break; - case 243: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 244: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 244: /* cmd ::= SHOW VNODES NK_STRING */ + case 245: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 245: /* db_name_cond_opt ::= */ - case 250: /* from_db_opt ::= */ yytestcase(yyruleno==250); -{ yymsp[1].minor.yy600 = createDefaultDatabaseCondValue(pCxt); } + case 246: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy42, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 246: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 247: /* cmd ::= SHOW CLUSTER ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 247: /* like_pattern_opt ::= */ - case 306: /* subtable_opt ::= */ yytestcase(yyruleno==306); - case 419: /* case_when_else_opt ::= */ yytestcase(yyruleno==419); - case 449: /* from_clause_opt ::= */ yytestcase(yyruleno==449); - case 478: /* where_clause_opt ::= */ yytestcase(yyruleno==478); - case 487: /* twindow_clause_opt ::= */ yytestcase(yyruleno==487); - case 492: /* sliding_opt ::= */ yytestcase(yyruleno==492); - case 494: /* fill_opt ::= */ yytestcase(yyruleno==494); - case 508: /* having_clause_opt ::= */ yytestcase(yyruleno==508); - case 510: /* range_opt ::= */ yytestcase(yyruleno==510); - case 512: /* every_opt ::= */ yytestcase(yyruleno==512); - case 525: /* slimit_clause_opt ::= */ yytestcase(yyruleno==525); - case 529: /* limit_clause_opt ::= */ yytestcase(yyruleno==529); -{ yymsp[1].minor.yy600 = NULL; } + case 248: /* db_name_cond_opt ::= */ + case 253: /* from_db_opt ::= */ yytestcase(yyruleno==253); +{ yymsp[1].minor.yy42 = createDefaultDatabaseCondValue(pCxt); } break; - case 248: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 249: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy42 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy225); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 249: /* table_name_cond ::= table_name */ -{ yylhsminor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 250: /* like_pattern_opt ::= */ + case 321: /* subtable_opt ::= */ yytestcase(yyruleno==321); + case 436: /* case_when_else_opt ::= */ yytestcase(yyruleno==436); + case 466: /* from_clause_opt ::= */ yytestcase(yyruleno==466); + case 495: /* where_clause_opt ::= */ yytestcase(yyruleno==495); + case 504: /* twindow_clause_opt ::= */ yytestcase(yyruleno==504); + case 510: /* sliding_opt ::= */ yytestcase(yyruleno==510); + case 512: /* fill_opt ::= */ yytestcase(yyruleno==512); + case 526: /* having_clause_opt ::= */ yytestcase(yyruleno==526); + case 528: /* range_opt ::= */ yytestcase(yyruleno==528); + case 530: /* every_opt ::= */ yytestcase(yyruleno==530); + case 543: /* slimit_clause_opt ::= */ yytestcase(yyruleno==543); + case 547: /* limit_clause_opt ::= */ yytestcase(yyruleno==547); +{ yymsp[1].minor.yy42 = NULL; } break; - case 251: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy600 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy77); } + case 251: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 255: /* tag_item ::= TBNAME */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 252: /* table_name_cond ::= table_name */ +{ yylhsminor.yy42 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy225); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 258: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 254: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy42 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy225); } break; - case 259: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy600 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy77), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 258: /* tag_item ::= TBNAME */ +{ yylhsminor.yy42 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 260: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy841, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } + case 261: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy42 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy225), &yymsp[0].minor.yy225); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 261: /* cmd ::= DROP INDEX exists_opt full_index_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy841, yymsp[0].minor.yy600); } + case 262: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy42 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy225), &yymsp[0].minor.yy225); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 262: /* full_index_name ::= index_name */ -{ yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy77); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 263: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy103, yymsp[-3].minor.yy42, yymsp[-1].minor.yy42, NULL, yymsp[0].minor.yy42); } break; - case 263: /* full_index_name ::= db_name NK_DOT index_name */ -{ yylhsminor.yy600 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 264: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy103, yymsp[-5].minor.yy42, yymsp[-3].minor.yy42, yymsp[-1].minor.yy110, NULL); } break; - case 264: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy600 = createIndexOption(pCxt, yymsp[-7].minor.yy601, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 265: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy103, yymsp[0].minor.yy42); } break; - case 265: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy600 = createIndexOption(pCxt, yymsp[-9].minor.yy601, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 266: /* full_index_name ::= index_name */ +{ yylhsminor.yy42 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy225); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 268: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy600 = createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 267: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy42 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 269: /* sma_stream_opt ::= */ - case 297: /* stream_options ::= */ yytestcase(yyruleno==297); -{ yymsp[1].minor.yy600 = createStreamOptions(pCxt); } + case 268: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy42 = createIndexOption(pCxt, yymsp[-7].minor.yy110, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), NULL, yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } break; - case 270: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - case 301: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==301); -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 269: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy42 = createIndexOption(pCxt, yymsp[-9].minor.yy110, releaseRawExprNode(pCxt, yymsp[-5].minor.yy42), releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } break; - case 271: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 272: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy42 = createFunctionNode(pCxt, &yymsp[-3].minor.yy225, yymsp[-1].minor.yy110); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 272: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - case 304: /* stream_options ::= stream_options DELETE_MARK duration_literal */ yytestcase(yyruleno==304); -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 278: /* sma_stream_opt ::= */ + case 312: /* stream_options ::= */ yytestcase(yyruleno==312); +{ yymsp[1].minor.yy42 = createStreamOptions(pCxt); } break; - case 273: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, yymsp[0].minor.yy600); } + case 279: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + case 316: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==316); +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 274: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy77, false); } + case 280: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 275: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[0].minor.yy77, true); } + case 281: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + case 319: /* stream_options ::= stream_options DELETE_MARK duration_literal */ yytestcase(yyruleno==319); +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 276: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy841, &yymsp[-3].minor.yy77, yymsp[0].minor.yy600, false); } + case 282: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy103, &yymsp[-2].minor.yy225, yymsp[0].minor.yy42); } break; - case 277: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy841, &yymsp[-5].minor.yy77, yymsp[0].minor.yy600, true); } + case 283: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy103, &yymsp[-3].minor.yy225, &yymsp[0].minor.yy225, false); } break; - case 278: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 284: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy103, &yymsp[-5].minor.yy225, &yymsp[0].minor.yy225, true); } break; - case 279: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy841, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77); } + case 285: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy103, &yymsp[-3].minor.yy225, yymsp[0].minor.yy42, false); } break; - case 280: /* cmd ::= DESC full_table_name */ - case 281: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==281); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy600); } + case 286: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy103, &yymsp[-5].minor.yy225, yymsp[0].minor.yy42, true); } break; - case 282: /* cmd ::= RESET QUERY CACHE */ + case 287: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy103, &yymsp[0].minor.yy225); } + break; + case 288: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy103, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225); } + break; + case 289: /* cmd ::= DESC full_table_name */ + case 290: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==290); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy42); } + break; + case 291: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 283: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy841, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 292: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 293: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==293); +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy103, yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } break; - case 286: /* explain_options ::= */ -{ yymsp[1].minor.yy600 = createDefaultExplainOptions(pCxt); } + case 296: /* explain_options ::= */ +{ yymsp[1].minor.yy42 = createDefaultExplainOptions(pCxt); } break; - case 287: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy600 = setExplainVerbose(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 297: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy42 = setExplainVerbose(pCxt, yymsp[-2].minor.yy42, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 288: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy600 = setExplainRatio(pCxt, yymsp[-2].minor.yy600, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 298: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy42 = setExplainRatio(pCxt, yymsp[-2].minor.yy42, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 289: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy841, yymsp[-8].minor.yy841, &yymsp[-5].minor.yy77, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy888, yymsp[0].minor.yy248); } + case 299: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy103, yymsp[-8].minor.yy103, &yymsp[-5].minor.yy225, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy448, yymsp[0].minor.yy508); } break; - case 290: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 300: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy103, &yymsp[0].minor.yy225); } break; - case 295: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name tags_def_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-8].minor.yy841, &yymsp[-7].minor.yy77, yymsp[-4].minor.yy600, yymsp[-6].minor.yy600, yymsp[-3].minor.yy601, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } + case 305: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy103, &yymsp[-8].minor.yy225, yymsp[-5].minor.yy42, yymsp[-7].minor.yy42, yymsp[-3].minor.yy110, yymsp[-2].minor.yy42, yymsp[0].minor.yy42, yymsp[-4].minor.yy110); } break; - case 296: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy841, &yymsp[0].minor.yy77); } + case 306: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy103, &yymsp[0].minor.yy225); } break; - case 298: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 313: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 299: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 314: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 300: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy600)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 315: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy42)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy42)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); yylhsminor.yy42 = yymsp[-3].minor.yy42; } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 302: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 317: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy42)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy42 = yymsp[-3].minor.yy42; } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 303: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-2].minor.yy600)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-2].minor.yy600; } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 318: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-2].minor.yy42)->fillHistory = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy42 = yymsp[-2].minor.yy42; } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 305: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy600)->ignoreUpdate = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy600 = yymsp[-3].minor.yy600; } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 320: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy42)->ignoreUpdate = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy42 = yymsp[-3].minor.yy42; } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 307: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 493: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==493); - case 513: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==513); -{ yymsp[-3].minor.yy600 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy600); } + case 322: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 511: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==511); + case 531: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==531); +{ yymsp[-3].minor.yy42 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy42); } break; - case 308: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 323: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 309: /* cmd ::= KILL QUERY NK_STRING */ + case 324: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 310: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 325: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 311: /* cmd ::= BALANCE VGROUP */ + case 326: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 312: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 327: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 313: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy601); } + case 328: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy110); } break; - case 314: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 329: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 315: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy601 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 317: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } - break; - case 319: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy600, yymsp[-2].minor.yy601, yymsp[0].minor.yy600); } - break; - case 320: /* cmd ::= INSERT INTO full_table_name query_or_subquery */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy600, NULL, yymsp[0].minor.yy600); } - break; - case 321: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 322: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 323: /* literal ::= NK_STRING */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 324: /* literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 325: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 326: /* literal ::= duration_literal */ - case 336: /* signed_literal ::= signed */ yytestcase(yyruleno==336); - case 357: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==357); - case 358: /* expression ::= literal */ yytestcase(yyruleno==358); - case 359: /* expression ::= pseudo_column */ yytestcase(yyruleno==359); - case 360: /* expression ::= column_reference */ yytestcase(yyruleno==360); - case 361: /* expression ::= function_expression */ yytestcase(yyruleno==361); - case 362: /* expression ::= case_when_expression */ yytestcase(yyruleno==362); - case 392: /* function_expression ::= literal_func */ yytestcase(yyruleno==392); - case 441: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==441); - case 445: /* boolean_primary ::= predicate */ yytestcase(yyruleno==445); - case 447: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==447); - case 448: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==448); - case 451: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==451); - case 453: /* table_reference ::= table_primary */ yytestcase(yyruleno==453); - case 454: /* table_reference ::= joined_table */ yytestcase(yyruleno==454); - case 458: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==458); - case 515: /* query_simple ::= query_specification */ yytestcase(yyruleno==515); - case 516: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==516); - case 519: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==519); - case 521: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==521); -{ yylhsminor.yy600 = yymsp[0].minor.yy600; } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 327: /* literal ::= NULL */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 328: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 329: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 330: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 331: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 332: /* signed ::= NK_MINUS NK_INTEGER */ + case 330: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy110 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 332: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } + break; + case 335: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ yymsp[-6].minor.yy42 = createInsertStmt(pCxt, yymsp[-4].minor.yy42, yymsp[-2].minor.yy110, yymsp[0].minor.yy42); } + break; + case 336: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +{ yymsp[-3].minor.yy42 = createInsertStmt(pCxt, yymsp[-1].minor.yy42, NULL, yymsp[0].minor.yy42); } + break; + case 337: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 338: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 339: /* literal ::= NK_STRING */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 340: /* literal ::= NK_BOOL */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 341: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; + break; + case 342: /* literal ::= duration_literal */ + case 352: /* signed_literal ::= signed */ yytestcase(yyruleno==352); + case 373: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==373); + case 374: /* expression ::= literal */ yytestcase(yyruleno==374); + case 375: /* expression ::= pseudo_column */ yytestcase(yyruleno==375); + case 376: /* expression ::= column_reference */ yytestcase(yyruleno==376); + case 377: /* expression ::= function_expression */ yytestcase(yyruleno==377); + case 378: /* expression ::= case_when_expression */ yytestcase(yyruleno==378); + case 409: /* function_expression ::= literal_func */ yytestcase(yyruleno==409); + case 458: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==458); + case 462: /* boolean_primary ::= predicate */ yytestcase(yyruleno==462); + case 464: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==464); + case 465: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==465); + case 468: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==468); + case 470: /* table_reference ::= table_primary */ yytestcase(yyruleno==470); + case 471: /* table_reference ::= joined_table */ yytestcase(yyruleno==471); + case 475: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==475); + case 533: /* query_simple ::= query_specification */ yytestcase(yyruleno==533); + case 534: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==534); + case 537: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==537); + case 539: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==539); +{ yylhsminor.yy42 = yymsp[0].minor.yy42; } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 343: /* literal ::= NULL */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 344: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 345: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 346: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 347: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 348: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 333: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 349: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 334: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 350: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 335: /* signed ::= NK_MINUS NK_FLOAT */ + case 351: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; - break; - case 337: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 338: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 339: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 340: /* signed_literal ::= duration_literal */ - case 342: /* signed_literal ::= literal_func */ yytestcase(yyruleno==342); - case 412: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==412); - case 474: /* select_item ::= common_expression */ yytestcase(yyruleno==474); - case 484: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==484); - case 520: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==520); - case 522: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==522); - case 535: /* search_condition ::= common_expression */ yytestcase(yyruleno==535); -{ yylhsminor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 341: /* signed_literal ::= NULL */ -{ yylhsminor.yy600 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 343: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy600 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 363: /* expression ::= NK_LP expression NK_RP */ - case 446: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==446); - case 534: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==534); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 364: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy42 = yylhsminor.yy42; + break; + case 353: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 354: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 355: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 356: /* signed_literal ::= duration_literal */ + case 358: /* signed_literal ::= literal_func */ yytestcase(yyruleno==358); + case 429: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==429); + case 491: /* select_item ::= common_expression */ yytestcase(yyruleno==491); + case 501: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==501); + case 538: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==538); + case 540: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==540); + case 553: /* search_condition ::= common_expression */ yytestcase(yyruleno==553); +{ yylhsminor.yy42 = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 357: /* signed_literal ::= NULL */ +{ yylhsminor.yy42 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 359: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy42 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 379: /* expression ::= NK_LP expression NK_RP */ + case 463: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==463); + case 552: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==552); +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy42)); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 380: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy42)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 365: /* expression ::= NK_MINUS expr_or_subquery */ + case 381: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy42), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 366: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 382: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 367: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 383: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 368: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 384: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 369: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 385: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 370: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 386: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 371: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 387: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 372: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 388: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 373: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 389: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 376: /* column_reference ::= column_name */ -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy77, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy77)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 377: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77, createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy77)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 378: /* pseudo_column ::= ROWTS */ - case 379: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==379); - case 381: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==381); - case 382: /* pseudo_column ::= QEND */ yytestcase(yyruleno==382); - case 383: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==383); - case 384: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==384); - case 385: /* pseudo_column ::= WEND */ yytestcase(yyruleno==385); - case 386: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==386); - case 387: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==387); - case 388: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==388); - case 394: /* literal_func ::= NOW */ yytestcase(yyruleno==394); -{ yylhsminor.yy600 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 380: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy77)))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 389: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 390: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==390); -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy77, yymsp[-1].minor.yy601)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 391: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy888)); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; - break; - case 393: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy77, NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 408: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy601 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy601 = yylhsminor.yy601; - break; - case 413: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 477: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==477); -{ yylhsminor.yy600 = createColumnNode(pCxt, &yymsp[-2].minor.yy77, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 414: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; - break; - case 415: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-2].minor.yy601, yymsp[-1].minor.yy600)); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; - break; - case 418: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy600 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600)); } - break; - case 420: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy600 = releaseRawExprNode(pCxt, yymsp[0].minor.yy600); } - break; - case 421: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 426: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==426); + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 392: /* column_reference ::= column_name */ +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy225, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy225)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 393: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225, createColumnNode(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy225)); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 394: /* pseudo_column ::= ROWTS */ + case 395: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==395); + case 397: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==397); + case 398: /* pseudo_column ::= QEND */ yytestcase(yyruleno==398); + case 399: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==399); + case 400: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==400); + case 401: /* pseudo_column ::= WEND */ yytestcase(yyruleno==401); + case 402: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==402); + case 403: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==403); + case 404: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==404); + case 405: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==405); + case 411: /* literal_func ::= NOW */ yytestcase(yyruleno==411); +{ yylhsminor.yy42 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy42 = yylhsminor.yy42; + break; + case 396: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy225)))); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 406: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 407: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==407); +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy225, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy225, yymsp[-1].minor.yy110)); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; + break; + case 408: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), yymsp[-1].minor.yy448)); } + yymsp[-5].minor.yy42 = yylhsminor.yy42; + break; + case 410: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy225, NULL)); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 425: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy110 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy110 = yylhsminor.yy110; + break; + case 430: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 494: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==494); +{ yylhsminor.yy42 = createColumnNode(pCxt, &yymsp[-2].minor.yy225, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; + break; + case 431: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy110, yymsp[-1].minor.yy42)); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; + break; + case 432: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), yymsp[-2].minor.yy110, yymsp[-1].minor.yy42)); } + yymsp[-4].minor.yy42 = yylhsminor.yy42; + break; + case 435: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy42 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42)); } + break; + case 437: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy42 = releaseRawExprNode(pCxt, yymsp[0].minor.yy42); } + break; + case 438: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 443: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==443); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy666, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy2, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 422: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 439: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy42), releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-4].minor.yy600 = yylhsminor.yy600; + yymsp[-4].minor.yy42 = yylhsminor.yy42; break; - case 423: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 440: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy42), releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + yymsp[-5].minor.yy42 = yylhsminor.yy42; break; - case 424: /* predicate ::= expr_or_subquery IS NULL */ + case 441: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), NULL)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 425: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 442: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), NULL)); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 427: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_THAN; } + case 444: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy2 = OP_TYPE_LOWER_THAN; } break; - case 428: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_THAN; } + case 445: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy2 = OP_TYPE_GREATER_THAN; } break; - case 429: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LOWER_EQUAL; } + case 446: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy2 = OP_TYPE_LOWER_EQUAL; } break; - case 430: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy666 = OP_TYPE_GREATER_EQUAL; } + case 447: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy2 = OP_TYPE_GREATER_EQUAL; } break; - case 431: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy666 = OP_TYPE_NOT_EQUAL; } + case 448: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy2 = OP_TYPE_NOT_EQUAL; } break; - case 432: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy666 = OP_TYPE_EQUAL; } + case 449: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy2 = OP_TYPE_EQUAL; } break; - case 433: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy666 = OP_TYPE_LIKE; } + case 450: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy2 = OP_TYPE_LIKE; } break; - case 434: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_LIKE; } + case 451: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy2 = OP_TYPE_NOT_LIKE; } break; - case 435: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_MATCH; } + case 452: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy2 = OP_TYPE_MATCH; } break; - case 436: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy666 = OP_TYPE_NMATCH; } + case 453: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy2 = OP_TYPE_NMATCH; } break; - case 437: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy666 = OP_TYPE_JSON_CONTAINS; } + case 454: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy2 = OP_TYPE_JSON_CONTAINS; } break; - case 438: /* in_op ::= IN */ -{ yymsp[0].minor.yy666 = OP_TYPE_IN; } + case 455: /* in_op ::= IN */ +{ yymsp[0].minor.yy2 = OP_TYPE_IN; } break; - case 439: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy666 = OP_TYPE_NOT_IN; } + case 456: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy2 = OP_TYPE_NOT_IN; } break; - case 440: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 457: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy110)); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 442: /* boolean_value_expression ::= NOT boolean_primary */ + case 459: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy600), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy42), NULL)); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 443: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 460: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 444: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 461: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy600); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy600); - yylhsminor.yy600 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy42); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy42); + yylhsminor.yy42 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 450: /* from_clause_opt ::= FROM table_reference_list */ - case 479: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==479); - case 509: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==509); -{ yymsp[-1].minor.yy600 = yymsp[0].minor.yy600; } + case 467: /* from_clause_opt ::= FROM table_reference_list */ + case 496: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==496); + case 527: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==527); +{ yymsp[-1].minor.yy42 = yymsp[0].minor.yy42; } break; - case 452: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy600, yymsp[0].minor.yy600, NULL); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 469: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy42 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy42, yymsp[0].minor.yy42, NULL); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 455: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 472: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy42 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy225, &yymsp[0].minor.yy225); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 456: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy600 = createRealTableNode(pCxt, &yymsp[-3].minor.yy77, &yymsp[-1].minor.yy77, &yymsp[0].minor.yy77); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 473: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy42 = createRealTableNode(pCxt, &yymsp[-3].minor.yy225, &yymsp[-1].minor.yy225, &yymsp[0].minor.yy225); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 457: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy600 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 474: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy42 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy42), &yymsp[0].minor.yy225); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 459: /* alias_opt ::= */ -{ yymsp[1].minor.yy77 = nil_token; } + case 476: /* alias_opt ::= */ +{ yymsp[1].minor.yy225 = nil_token; } break; - case 461: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy77 = yymsp[0].minor.yy77; } + case 478: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy225 = yymsp[0].minor.yy225; } break; - case 462: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 463: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==463); -{ yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; } + case 479: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 480: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==480); +{ yymsp[-2].minor.yy42 = yymsp[-1].minor.yy42; } break; - case 464: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy600 = createJoinTableNode(pCxt, yymsp[-4].minor.yy560, yymsp[-5].minor.yy600, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-5].minor.yy600 = yylhsminor.yy600; + case 481: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy42 = createJoinTableNode(pCxt, yymsp[-4].minor.yy638, yymsp[-5].minor.yy42, yymsp[-2].minor.yy42, yymsp[0].minor.yy42); } + yymsp[-5].minor.yy42 = yylhsminor.yy42; break; - case 465: /* join_type ::= */ -{ yymsp[1].minor.yy560 = JOIN_TYPE_INNER; } + case 482: /* join_type ::= */ +{ yymsp[1].minor.yy638 = JOIN_TYPE_INNER; } break; - case 466: /* join_type ::= INNER */ -{ yymsp[0].minor.yy560 = JOIN_TYPE_INNER; } + case 483: /* join_type ::= INNER */ +{ yymsp[0].minor.yy638 = JOIN_TYPE_INNER; } break; - case 467: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 484: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy600 = createSelectStmt(pCxt, yymsp[-10].minor.yy841, yymsp[-9].minor.yy601, yymsp[-8].minor.yy600); - yymsp[-11].minor.yy600 = addWhereClause(pCxt, yymsp[-11].minor.yy600, yymsp[-7].minor.yy600); - yymsp[-11].minor.yy600 = addPartitionByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-6].minor.yy601); - yymsp[-11].minor.yy600 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy600, yymsp[-2].minor.yy600); - yymsp[-11].minor.yy600 = addGroupByClause(pCxt, yymsp[-11].minor.yy600, yymsp[-1].minor.yy601); - yymsp[-11].minor.yy600 = addHavingClause(pCxt, yymsp[-11].minor.yy600, yymsp[0].minor.yy600); - yymsp[-11].minor.yy600 = addRangeClause(pCxt, yymsp[-11].minor.yy600, yymsp[-5].minor.yy600); - yymsp[-11].minor.yy600 = addEveryClause(pCxt, yymsp[-11].minor.yy600, yymsp[-4].minor.yy600); - yymsp[-11].minor.yy600 = addFillClause(pCxt, yymsp[-11].minor.yy600, yymsp[-3].minor.yy600); + yymsp[-11].minor.yy42 = createSelectStmt(pCxt, yymsp[-10].minor.yy103, yymsp[-9].minor.yy110, yymsp[-8].minor.yy42); + yymsp[-11].minor.yy42 = addWhereClause(pCxt, yymsp[-11].minor.yy42, yymsp[-7].minor.yy42); + yymsp[-11].minor.yy42 = addPartitionByClause(pCxt, yymsp[-11].minor.yy42, yymsp[-6].minor.yy110); + yymsp[-11].minor.yy42 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy42, yymsp[-2].minor.yy42); + yymsp[-11].minor.yy42 = addGroupByClause(pCxt, yymsp[-11].minor.yy42, yymsp[-1].minor.yy110); + yymsp[-11].minor.yy42 = addHavingClause(pCxt, yymsp[-11].minor.yy42, yymsp[0].minor.yy42); + yymsp[-11].minor.yy42 = addRangeClause(pCxt, yymsp[-11].minor.yy42, yymsp[-5].minor.yy42); + yymsp[-11].minor.yy42 = addEveryClause(pCxt, yymsp[-11].minor.yy42, yymsp[-4].minor.yy42); + yymsp[-11].minor.yy42 = addFillClause(pCxt, yymsp[-11].minor.yy42, yymsp[-3].minor.yy42); } break; - case 470: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy841 = false; } + case 487: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy103 = false; } + break; + case 490: /* select_item ::= NK_STAR */ +{ yylhsminor.yy42 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy42 = yylhsminor.yy42; break; - case 473: /* select_item ::= NK_STAR */ -{ yylhsminor.yy600 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 492: /* select_item ::= common_expression column_alias */ + case 502: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==502); +{ yylhsminor.yy42 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy42), &yymsp[0].minor.yy225); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; - case 475: /* select_item ::= common_expression column_alias */ - case 485: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==485); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 493: /* select_item ::= common_expression AS column_alias */ + case 503: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==503); +{ yylhsminor.yy42 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), &yymsp[0].minor.yy225); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 476: /* select_item ::= common_expression AS column_alias */ - case 486: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==486); -{ yylhsminor.yy600 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), &yymsp[0].minor.yy77); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 498: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 523: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==523); + case 542: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==542); +{ yymsp[-2].minor.yy110 = yymsp[0].minor.yy110; } break; - case 481: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 505: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==505); - case 524: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==524); -{ yymsp[-2].minor.yy601 = yymsp[0].minor.yy601; } + case 505: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy42 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), releaseRawExprNode(pCxt, yymsp[-1].minor.yy42)); } break; - case 488: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy600 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 506: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy42 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy42)); } break; - case 489: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy600 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 507: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy42 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), NULL, yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } break; - case 490: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), NULL, yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 508: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy42 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy42), releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), yymsp[-1].minor.yy42, yymsp[0].minor.yy42); } break; - case 491: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy600 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy600), releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), yymsp[-1].minor.yy600, yymsp[0].minor.yy600); } + case 509: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy42 = createEventWindowNode(pCxt, yymsp[-3].minor.yy42, yymsp[0].minor.yy42); } break; - case 495: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy600 = createFillNode(pCxt, yymsp[-1].minor.yy798, NULL); } + case 513: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy42 = createFillNode(pCxt, yymsp[-1].minor.yy410, NULL); } break; - case 496: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } + case 514: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy42 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy110)); } break; - case 497: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy600 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy601)); } + case 515: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy42 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy110)); } break; - case 498: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy798 = FILL_MODE_NONE; } + case 516: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy410 = FILL_MODE_NONE; } break; - case 499: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy798 = FILL_MODE_PREV; } + case 517: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy410 = FILL_MODE_PREV; } break; - case 500: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy798 = FILL_MODE_NULL; } + case 518: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy410 = FILL_MODE_NULL; } break; - case 501: /* fill_mode ::= NULL_F */ -{ yymsp[0].minor.yy798 = FILL_MODE_NULL_F; } + case 519: /* fill_mode ::= NULL_F */ +{ yymsp[0].minor.yy410 = FILL_MODE_NULL_F; } break; - case 502: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy798 = FILL_MODE_LINEAR; } + case 520: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy410 = FILL_MODE_LINEAR; } break; - case 503: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy798 = FILL_MODE_NEXT; } + case 521: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy410 = FILL_MODE_NEXT; } break; - case 506: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy601 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[0].minor.yy601 = yylhsminor.yy601; + case 524: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy110 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } + yymsp[0].minor.yy110 = yylhsminor.yy110; break; - case 507: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy601 = addNodeToList(pCxt, yymsp[-2].minor.yy601, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy600))); } - yymsp[-2].minor.yy601 = yylhsminor.yy601; + case 525: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy110 = addNodeToList(pCxt, yymsp[-2].minor.yy110, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy42))); } + yymsp[-2].minor.yy110 = yylhsminor.yy110; break; - case 511: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy600 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy600), releaseRawExprNode(pCxt, yymsp[-1].minor.yy600)); } + case 529: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy42 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy42), releaseRawExprNode(pCxt, yymsp[-1].minor.yy42)); } break; - case 514: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 532: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy600 = addOrderByClause(pCxt, yymsp[-3].minor.yy600, yymsp[-2].minor.yy601); - yylhsminor.yy600 = addSlimitClause(pCxt, yylhsminor.yy600, yymsp[-1].minor.yy600); - yylhsminor.yy600 = addLimitClause(pCxt, yylhsminor.yy600, yymsp[0].minor.yy600); + yylhsminor.yy42 = addOrderByClause(pCxt, yymsp[-3].minor.yy42, yymsp[-2].minor.yy110); + yylhsminor.yy42 = addSlimitClause(pCxt, yylhsminor.yy42, yymsp[-1].minor.yy42); + yylhsminor.yy42 = addLimitClause(pCxt, yylhsminor.yy42, yymsp[0].minor.yy42); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 517: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-3].minor.yy600 = yylhsminor.yy600; + case 535: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy42 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy42, yymsp[0].minor.yy42); } + yymsp[-3].minor.yy42 = yylhsminor.yy42; break; - case 518: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy600 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 536: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy42 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy42, yymsp[0].minor.yy42); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 526: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 530: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==530); -{ yymsp[-1].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 544: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 548: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==548); +{ yymsp[-1].minor.yy42 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 527: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 531: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==531); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 545: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 549: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==549); +{ yymsp[-3].minor.yy42 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 528: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 532: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==532); -{ yymsp[-3].minor.yy600 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 546: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 550: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==550); +{ yymsp[-3].minor.yy42 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 533: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy600 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy600); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 551: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy42 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy42); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 538: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy600 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy600), yymsp[-1].minor.yy32, yymsp[0].minor.yy385); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 556: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy42 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy42), yymsp[-1].minor.yy106, yymsp[0].minor.yy599); } + yymsp[-2].minor.yy42 = yylhsminor.yy42; break; - case 539: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy32 = ORDER_ASC; } + case 557: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy106 = ORDER_ASC; } break; - case 540: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy32 = ORDER_ASC; } + case 558: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy106 = ORDER_ASC; } break; - case 541: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy32 = ORDER_DESC; } + case 559: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy106 = ORDER_DESC; } break; - case 542: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy385 = NULL_ORDER_DEFAULT; } + case 560: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy599 = NULL_ORDER_DEFAULT; } break; - case 543: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_FIRST; } + case 561: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy599 = NULL_ORDER_FIRST; } break; - case 544: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy385 = NULL_ORDER_LAST; } + case 562: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy599 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index ae702ec02f1570ab9d92976abdb18059cd727175..c3f6c3ac72c70b799500b1e70cc31b72e585d575 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -102,6 +102,10 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_COLS, TSDB_SYSTEM_TABLE, 2) + .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_PRIVILEGES, TSDB_SYSTEM_TABLE, 2) .addColumn("user_name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN) .addColumn("privilege", TSDB_DATA_TYPE_BINARY, 10) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9cc55a7cd55a0060635e9a9044ac076ce5a77119..71b7c1a678b92f2ba8f7b16db10f55e949af1bbe 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -33,7 +33,6 @@ std::unique_ptr g_mockCatalogService; class TableBuilder : public ITableBuilder { public: virtual TableBuilder& addColumn(const string& name, int8_t type, int32_t bytes) { - assert(colId_ <= schema()->tableInfo.numOfTags + schema()->tableInfo.numOfColumns); SSchema* col = schema()->schema + (colId_ - 1); col->type = type; col->colId = colId_++; @@ -332,7 +331,7 @@ class MockCatalogServiceImpl { info.dstTbUid = getNextId(); info.dstVgId = pReq->dstVgId; genEpSet(&info.epSet); - info.expr = strdup(pReq->expr); + info.expr = taosStrdup(pReq->expr); auto it = index_.find(pReq->stb); if (index_.end() == it) { index_.insert(std::make_pair(string(pReq->stb), std::vector{info})); @@ -375,7 +374,7 @@ class MockCatalogServiceImpl { STableIndexInfo* copyTableIndexInfo(STableIndexInfo* pDst, const STableIndexInfo* pSrc) const { memcpy(pDst, pSrc, sizeof(STableIndexInfo)); - pDst->expr = strdup(pSrc->expr); + pDst->expr = taosStrdup(pSrc->expr); return pDst; } @@ -427,7 +426,7 @@ class MockCatalogServiceImpl { int32_t copyTableSchemaMeta(const string& db, const string& tbname, std::unique_ptr* dst) const { STableMeta* src = getTableSchemaMeta(db, tbname); if (nullptr == src) { - return TSDB_CODE_TSC_INVALID_TABLE_NAME; + return TSDB_CODE_PAR_TABLE_NOT_EXIST; } int32_t len = sizeof(STableMeta) + sizeof(SSchema) * (src->tableInfo.numOfTags + src->tableInfo.numOfColumns); dst->reset((STableMeta*)taosMemoryCalloc(1, len)); diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index ba1ba9fc831ec8661cc7a8252a1772d5bbae6317..bfcf6ec27e7eec142a4a586bc31c61a8690b51a3 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -21,6 +21,25 @@ namespace ParserTest { class ParserInitialATest : public ParserDdlTest {}; +/* + * ALTER ACCOUNT account_name alter_account_options + * + * alter_account_options: + * alter_account_option ... + * + * alter_account_option: { + * PASS value + * | PPS value + * | TSERIES value + * | STORAGE value + * | STREAMS value + * | QTIME value + * | DBS value + * | USERS value + * | CONNS value + * | STATE value + * } + */ TEST_F(ParserInitialATest, alterAccount) { useDb("root", "test"); @@ -48,6 +67,7 @@ TEST_F(ParserInitialATest, alterDnode) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DNODE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_CONFIG_DNODE); SMCfgDnodeReq req = {0}; ASSERT_EQ(tDeserializeSMCfgDnodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(req.dnodeId, expect.dnodeId); @@ -86,9 +106,8 @@ TEST_F(ParserInitialATest, alterDnode) { * | KEEP {int_value | duration_value} -- rang [1, 365000], default 3650, unit day * | PAGES int_value -- rang [64, INT32_MAX], default 256, unit page * | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica - * | STRICT {'off' | 'on'} -- todo: default 'off' * | WAL_LEVEL int_value -- enum 1, 2, default 1 - * | SST_TRIGGER int_value -- rang [1, 16], default 8 + * | STT_TRIGGER int_value -- rang [1, 16], default 8 * } */ TEST_F(ParserInitialATest, alterDatabase) { @@ -130,10 +149,11 @@ TEST_F(ParserInitialATest, alterDatabase) { auto setAlterDbStrict = [&](int8_t strict) { expect.strict = strict; }; auto setAlterDbCacheModel = [&](int8_t cacheModel) { expect.cacheLast = cacheModel; }; auto setAlterDbReplica = [&](int8_t replications) { expect.replications = replications; }; - auto setAlterDbSstTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; + auto setAlterDbSttTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DATABASE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_DB); SAlterDbReq req = {0}; ASSERT_EQ(tDeserializeSAlterDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.db), std::string(expect.db)); @@ -161,11 +181,12 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); - setAlterDbSstTrigger(16); + setAlterDbSttTrigger(16); setAlterDbBuffer(16); setAlterDbPages(128); + setAlterDbReplica(3); run("ALTER DATABASE test BUFFER 16 CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 PAGES 128 " - "WAL_LEVEL 1 STT_TRIGGER 16"); + "REPLICA 3 WAL_LEVEL 1 STT_TRIGGER 16"); clearAlterDbReq(); initAlterDb("test"); @@ -240,6 +261,22 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbWal(2); run("ALTER DATABASE test WAL_LEVEL 2"); clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbReplica(1); + run("ALTER DATABASE test REPLICA 1"); + setAlterDbReplica(3); + run("ALTER DATABASE test REPLICA 3"); + clearAlterDbReq(); + + initAlterDb("test"); + setAlterDbSttTrigger(1); + run("ALTER DATABASE test STT_TRIGGER 1"); + setAlterDbSttTrigger(4); + run("ALTER DATABASE test STT_TRIGGER 4"); + setAlterDbSttTrigger(16); + run("ALTER DATABASE test STT_TRIGGER 16"); + clearAlterDbReq(); } TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { @@ -260,6 +297,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { run("ALTER DATABASE test PAGES 63", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION); + run("ALTER DATABASE test REPLICA 2", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test STT_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test STT_TRIGGER 17", TSDB_CODE_PAR_INVALID_DB_OPTION); // Regardless of the specific sentence @@ -267,7 +305,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { } /* - * ALTER LOCAL dnode_id 'config' ['value'] + * ALTER LOCAL 'config' ['value'] */ TEST_F(ParserInitialATest, alterLocal) { useDb("root", "test"); @@ -311,19 +349,19 @@ TEST_F(ParserInitialATest, alterLocal) { * | ADD COLUMN col_name column_type * | DROP COLUMN col_name * | MODIFY COLUMN col_name column_type - * | RENAME COLUMN old_col_name new_col_name -- normal table - * | ADD TAG tag_name tag_type -- super table - * | DROP TAG tag_name -- super table - * | MODIFY TAG tag_name tag_type -- super table - * | RENAME TAG old_tag_name new_tag_name -- super table - * | SET TAG tag_name = new_tag_value -- child table + * | RENAME COLUMN old_col_name new_col_name -- only normal table + * | ADD TAG tag_name tag_type -- only super table + * | DROP TAG tag_name -- only super table + * | MODIFY TAG tag_name tag_type -- only super table + * | RENAME TAG old_tag_name new_tag_name -- only super table + * | SET TAG tag_name = new_tag_value -- only child table * } * * alter_table_options: * alter_table_option ... * * alter_table_option: { - * TTL int_value -- child/normal table + * TTL int_value -- only child/normal table * | COMMENT 'string_value' * } */ @@ -344,7 +382,7 @@ TEST_F(ParserInitialATest, alterSTable) { expect.name[len] = '\0'; expect.alterType = alterType; if (nullptr != pComment) { - expect.comment = strdup(pComment); + expect.comment = taosStrdup(pComment); expect.commentLen = strlen(pComment); } @@ -379,6 +417,7 @@ TEST_F(ParserInitialATest, alterSTable) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_SUPER_TABLE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_STB); SMAlterStbReq req = {0}; ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); ASSERT_EQ(std::string(req.name), std::string(expect.name)); @@ -444,136 +483,255 @@ TEST_F(ParserInitialATest, alterSTableSemanticCheck) { run("ALTER STABLE st1 TTL 10", TSDB_CODE_PAR_INVALID_ALTER_TABLE); } +/* + * ALTER TABLE [db_name.]tb_name alter_table_clause + * + * alter_table_clause: { + * alter_table_options + * | ADD COLUMN col_name column_type + * | DROP COLUMN col_name + * | MODIFY COLUMN col_name column_type + * | RENAME COLUMN old_col_name new_col_name -- only normal table + * | ADD TAG tag_name tag_type -- only super table + * | DROP TAG tag_name -- only super table + * | MODIFY TAG tag_name tag_type -- only super table + * | RENAME TAG old_tag_name new_tag_name -- only super table + * | SET TAG tag_name = new_tag_value -- only child table + * } + * + * alter_table_options: + * alter_table_option ... + * + * alter_table_option: { + * TTL int_value -- only child/normal table + * | COMMENT 'string_value' + * } + */ TEST_F(ParserInitialATest, alterTable) { useDb("root", "test"); - SVAlterTbReq expect = {0}; - - auto clearAlterTbReq = [&]() { - free(expect.tbName); - free(expect.colName); - free(expect.colNewName); - free(expect.tagName); - memset(&expect, 0, sizeof(SVAlterTbReq)); - }; - - auto setAlterTableCol = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0, - int32_t dataBytes = 0, const char* pNewColName = nullptr) { - expect.tbName = strdup(pTbname); - expect.action = alterType; - expect.colName = strdup(pColName); - - switch (alterType) { - case TSDB_ALTER_TABLE_ADD_COLUMN: - expect.type = dataType; - expect.flags = COL_SMA_ON; - expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0); - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: - expect.colModBytes = dataBytes; - break; - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - expect.colNewName = strdup(pNewColName); - break; - default: - break; - } - }; - - auto setAlterTableTag = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { - expect.tbName = strdup(pTbname); - expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; - expect.tagName = strdup(pTagName); - - expect.isNull = (nullptr == pNewVal); - expect.nTagVal = bytes; - expect.pTagVal = pNewVal; - }; - - auto setAlterTableOptions = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { - expect.tbName = strdup(pTbname); - expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; - if (-1 != ttl) { - expect.updateTTL = true; - expect.newTTL = ttl; - } - if (nullptr != pComment) { - expect.newCommentLen = strlen(pComment); - expect.newComment = pComment; - } - }; - - setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { - ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIF_STMT); - SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; - - ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); - ASSERT_NE(pStmt->pDataBlocks, nullptr); - ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1); - SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0); - void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); - ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); - - ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); - ASSERT_EQ(req.action, expect.action); - if (nullptr != expect.colName) { - ASSERT_EQ(std::string(req.colName), std::string(expect.colName)); - } - ASSERT_EQ(req.type, expect.type); - ASSERT_EQ(req.flags, expect.flags); - ASSERT_EQ(req.bytes, expect.bytes); - ASSERT_EQ(req.colModBytes, expect.colModBytes); - if (nullptr != expect.colNewName) { - ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName)); - } - if (nullptr != expect.tagName) { - ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName)); - } - ASSERT_EQ(req.isNull, expect.isNull); - ASSERT_EQ(req.nTagVal, expect.nTagVal); - ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); - ASSERT_EQ(req.updateTTL, expect.updateTTL); - ASSERT_EQ(req.newTTL, expect.newTTL); - if (nullptr != expect.newComment) { - ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment)); - ASSERT_EQ(req.newCommentLen, strlen(req.newComment)); - ASSERT_EQ(expect.newCommentLen, strlen(expect.newComment)); - } - - tDecoderClear(&coder); - }); - - setAlterTableOptions("t1", 10, nullptr); - run("ALTER TABLE t1 TTL 10"); - clearAlterTbReq(); - - setAlterTableOptions("t1", -1, (char*)"test"); - run("ALTER TABLE t1 COMMENT 'test'"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); - run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); - run("ALTER TABLE t1 DROP COLUMN c1"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE); - run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)"); - clearAlterTbReq(); - - setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); - run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); - clearAlterTbReq(); - - int32_t val = 10; - setAlterTableTag("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); - run("ALTER TABLE st1s1 SET TAG tag1=10"); - clearAlterTbReq(); + // normal/child table + { + SVAlterTbReq expect = {0}; + + auto clearAlterTbReq = [&]() { + free(expect.tbName); + free(expect.colName); + free(expect.colNewName); + free(expect.tagName); + memset(&expect, 0, sizeof(SVAlterTbReq)); + }; + + auto setAlterTableCol = [&](const char* pTbname, int8_t alterType, const char* pColName, int8_t dataType = 0, + int32_t dataBytes = 0, const char* pNewColName = nullptr) { + expect.tbName = strdup(pTbname); + expect.action = alterType; + expect.colName = strdup(pColName); + + switch (alterType) { + case TSDB_ALTER_TABLE_ADD_COLUMN: + expect.type = dataType; + expect.flags = COL_SMA_ON; + expect.bytes = dataBytes > 0 ? dataBytes : (dataType > 0 ? tDataTypes[dataType].bytes : 0); + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: + expect.colModBytes = dataBytes; + break; + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: + expect.colNewName = strdup(pNewColName); + break; + default: + break; + } + }; + + auto setAlterTableTag = [&](const char* pTbname, const char* pTagName, uint8_t* pNewVal, uint32_t bytes) { + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_TAG_VAL; + expect.tagName = strdup(pTagName); + + expect.isNull = (nullptr == pNewVal); + expect.nTagVal = bytes; + expect.pTagVal = pNewVal; + }; + + auto setAlterTableOptions = [&](const char* pTbname, int32_t ttl, char* pComment = nullptr) { + expect.tbName = strdup(pTbname); + expect.action = TSDB_ALTER_TABLE_UPDATE_OPTIONS; + if (-1 != ttl) { + expect.updateTTL = true; + expect.newTTL = ttl; + } + if (nullptr != pComment) { + expect.newCommentLen = strlen(pComment); + expect.newComment = pComment; + } + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIFY_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + + ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_ALTER_TABLE_STMT); + ASSERT_NE(pStmt->pDataBlocks, nullptr); + ASSERT_EQ(taosArrayGetSize(pStmt->pDataBlocks), 1); + SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, 0); + void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); + ASSERT_EQ(tDecodeSVAlterTbReq(&coder, &req), TSDB_CODE_SUCCESS); + + ASSERT_EQ(std::string(req.tbName), std::string(expect.tbName)); + ASSERT_EQ(req.action, expect.action); + if (nullptr != expect.colName) { + ASSERT_EQ(std::string(req.colName), std::string(expect.colName)); + } + ASSERT_EQ(req.type, expect.type); + ASSERT_EQ(req.flags, expect.flags); + ASSERT_EQ(req.bytes, expect.bytes); + ASSERT_EQ(req.colModBytes, expect.colModBytes); + if (nullptr != expect.colNewName) { + ASSERT_EQ(std::string(req.colNewName), std::string(expect.colNewName)); + } + if (nullptr != expect.tagName) { + ASSERT_EQ(std::string(req.tagName), std::string(expect.tagName)); + } + ASSERT_EQ(req.isNull, expect.isNull); + ASSERT_EQ(req.nTagVal, expect.nTagVal); + ASSERT_EQ(memcmp(req.pTagVal, expect.pTagVal, expect.nTagVal), 0); + ASSERT_EQ(req.updateTTL, expect.updateTTL); + ASSERT_EQ(req.newTTL, expect.newTTL); + if (nullptr != expect.newComment) { + ASSERT_EQ(std::string(req.newComment), std::string(expect.newComment)); + ASSERT_EQ(req.newCommentLen, strlen(req.newComment)); + ASSERT_EQ(expect.newCommentLen, strlen(expect.newComment)); + } + + tDecoderClear(&coder); + }); + + setAlterTableOptions("t1", 10, nullptr); + run("ALTER TABLE t1 TTL 10"); + clearAlterTbReq(); + + setAlterTableOptions("t1", -1, (char*)"test"); + run("ALTER TABLE t1 COMMENT 'test'"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_ADD_COLUMN, "cc1", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE t1 ADD COLUMN cc1 BIGINT"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_DROP_COLUMN, "c1"); + run("ALTER TABLE t1 DROP COLUMN c1"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, "c2", TSDB_DATA_TYPE_VARCHAR, 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE t1 MODIFY COLUMN c2 VARCHAR(30)"); + clearAlterTbReq(); + + setAlterTableCol("t1", TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, "c1", 0, 0, "cc1"); + run("ALTER TABLE t1 RENAME COLUMN c1 cc1"); + clearAlterTbReq(); + + int32_t val = 10; + setAlterTableTag("st1s1", "tag1", (uint8_t*)&val, sizeof(val)); + run("ALTER TABLE st1s1 SET TAG tag1=10"); + clearAlterTbReq(); + } + + // super table + { + SMAlterStbReq expect = {0}; + + auto clearAlterStbReq = [&]() { + tFreeSMAltertbReq(&expect); + memset(&expect, 0, sizeof(SMAlterStbReq)); + }; + + auto setAlterStbReq = [&](const char* pTbname, int8_t alterType, int32_t numOfFields = 0, + const char* pField1Name = nullptr, int8_t field1Type = 0, int32_t field1Bytes = 0, + const char* pField2Name = nullptr, const char* pComment = nullptr) { + int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname); + expect.name[len] = '\0'; + expect.alterType = alterType; + if (nullptr != pComment) { + expect.comment = strdup(pComment); + expect.commentLen = strlen(pComment); + } + + expect.numOfFields = numOfFields; + if (NULL == expect.pFields) { + expect.pFields = taosArrayInit(2, sizeof(TAOS_FIELD)); + TAOS_FIELD field = {0}; + taosArrayPush(expect.pFields, &field); + taosArrayPush(expect.pFields, &field); + } + + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + if (NULL != pField1Name) { + strcpy(pField->name, pField1Name); + pField->name[strlen(pField1Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = field1Type; + pField->bytes = field1Bytes > 0 ? field1Bytes : (field1Type > 0 ? tDataTypes[field1Type].bytes : 0); + + pField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + if (NULL != pField2Name) { + strcpy(pField->name, pField2Name); + pField->name[strlen(pField2Name)] = '\0'; + } else { + memset(pField, 0, sizeof(TAOS_FIELD)); + } + pField->type = 0; + pField->bytes = 0; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_TABLE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_STB); + SMAlterStbReq req = {0}; + ASSERT_EQ(tDeserializeSMAlterStbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(req.numOfFields, expect.numOfFields); + if (expect.numOfFields > 0) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 0); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 0); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + if (expect.numOfFields > 1) { + TAOS_FIELD* pField = (TAOS_FIELD*)taosArrayGet(req.pFields, 1); + TAOS_FIELD* pExpectField = (TAOS_FIELD*)taosArrayGet(expect.pFields, 1); + ASSERT_EQ(std::string(pField->name), std::string(pExpectField->name)); + ASSERT_EQ(pField->type, pExpectField->type); + ASSERT_EQ(pField->bytes, pExpectField->bytes); + } + tFreeSMAltertbReq(&req); + }); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_ADD_TAG, 1, "tag11", TSDB_DATA_TYPE_BIGINT); + run("ALTER TABLE st1 ADD TAG tag11 BIGINT"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_DROP_TAG, 1, "tag1"); + run("ALTER TABLE st1 DROP TAG tag1"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, 1, "tag2", TSDB_DATA_TYPE_VARCHAR, + 30 + VARSTR_HEADER_SIZE); + run("ALTER TABLE st1 MODIFY TAG tag2 VARCHAR(30)"); + clearAlterStbReq(); + + setAlterStbReq("st1", TSDB_ALTER_TABLE_UPDATE_TAG_NAME, 2, "tag1", 0, 0, "tag11"); + run("ALTER TABLE st1 RENAME TAG tag1 tag11"); + clearAlterStbReq(); + } } TEST_F(ParserInitialATest, alterTableSemanticCheck) { @@ -588,7 +746,7 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) { } /* - * ALTER USER user_name PASS str_value + * ALTER USER user_name alter_user_clause * * alter_user_clause: { * PASS str_value @@ -618,6 +776,7 @@ TEST_F(ParserInitialATest, alterUser) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_USER_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER); SAlterUserReq req = {0}; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); diff --git a/source/libs/parser/test/parExplainToSyncdbTest.cpp b/source/libs/parser/test/parExplainToSyncdbTest.cpp index f88d6d316b13606ba95226e669a59bcc588bf566..f23685b3058b911f23baf8c37c6608ab989de5c7 100644 --- a/source/libs/parser/test/parExplainToSyncdbTest.cpp +++ b/source/libs/parser/test/parExplainToSyncdbTest.cpp @@ -52,8 +52,8 @@ TEST_F(ParserExplainToSyncdbTest, grant) { ASSERT_EQ(string(req.objname), string(expect.objname)); }); - setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.test"); - run("GRANT ALL ON test.* TO wxy"); + setAlterUserReq(TSDB_ALTER_USER_ADD_ALL_DB, "wxy", "0.*"); + run("GRANT ALL ON *.* TO wxy"); setAlterUserReq(TSDB_ALTER_USER_ADD_READ_DB, "wxy", "0.test"); run("GRANT READ ON test.* TO wxy"); @@ -138,10 +138,38 @@ TEST_F(ParserExplainToSyncdbTest, redistributeVgroup) { TEST_F(ParserExplainToSyncdbTest, revoke) { useDb("root", "test"); - run("REVOKE ALL ON test.* FROM wxy"); + SAlterUserReq expect = {0}; + + auto setAlterUserReq = [&](int8_t alterType, const string& user, const string& obj) { + expect.alterType = alterType; + snprintf(expect.user, sizeof(expect.user), "%s", user.c_str()); + snprintf(expect.objname, sizeof(expect.objname), "%s", obj.c_str()); + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_REVOKE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_ALTER_USER); + SAlterUserReq req = {0}; + ASSERT_EQ(tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(string(req.user), string(expect.user)); + ASSERT_EQ(string(req.objname), string(expect.objname)); + }); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_ALL_DB, "wxy", "0.*"); + run("REVOKE ALL ON *.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_READ_DB, "wxy", "0.test"); run("REVOKE READ ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_WRITE_DB, "wxy", "0.test"); run("REVOKE WRITE ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_ALL_DB, "wxy", "0.test"); run("REVOKE READ, WRITE ON test.* FROM wxy"); + + setAlterUserReq(TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC, "wxy", "0.tp1"); + run("REVOKE SUBSCRIBE ON tp1 FROM wxy"); } // todo syncdb diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index eb264687f12ee4f5fc4777478d7b199eef9dc25b..8ba5802ad69598313e9274e8856988ab703cf3ae 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -21,12 +21,55 @@ namespace ParserTest { class ParserInitialCTest : public ParserDdlTest {}; +/* + * COMPACT DATABASE db_name + */ +TEST_F(ParserInitialCTest, compact) { + SCompactDbReq expect = {0}; + + auto setCompactDbReq = [&](const char* pDb) { snprintf(expect.db, sizeof(expect.db), "0.%s", pDb); }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_COMPACT_DATABASE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_COMPACT_DB); + SCompactDbReq req = {0}; + ASSERT_EQ(tDeserializeSCompactDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(std::string(req.db), std::string(expect.db)); + }); + + setCompactDbReq("wxy_db"); + run("COMPACT DATABASE wxy_db"); +} + +/* + * CREATE ACCOUNT account_name PASS value [create_account_options] + * + * create_account_options: + * create_account_option ... + * + * create_account_option: { + * PPS value + * | TSERIES value + * | STORAGE value + * | STREAMS value + * | QTIME value + * | DBS value + * | USERS value + * | CONNS value + * | STATE value + * } + */ TEST_F(ParserInitialCTest, createAccount) { useDb("root", "test"); run("CREATE ACCOUNT ac_wxy PASS '123456'", TSDB_CODE_PAR_EXPRIE_STATEMENT, PARSER_STAGE_PARSE); } +/* + * CREATE BNODE ON DNODE dnode_id + * the server does not support it temporarily + */ + /* * CREATE DATABASE [IF NOT EXISTS] db_name [database_options] * @@ -68,7 +111,7 @@ TEST_F(ParserInitialCTest, createDatabase) { memset(&expect, 0, sizeof(SCreateDbReq)); }; - auto setCreateDbReqFunc = [&](const char* pDbname, int8_t igExists = 0) { + auto setCreateDbReq = [&](const char* pDbname, int8_t igExists = 0) { int32_t len = snprintf(expect.db, sizeof(expect.db), "0.%s", pDbname); expect.db[len] = '\0'; expect.ignoreExist = igExists; @@ -102,28 +145,28 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; }; - auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; - auto setDbCachelastFunc = [&](int8_t cachelast) { expect.cacheLast = cachelast; }; + auto setDbBuffer = [&](int32_t buffer) { expect.buffer = buffer; }; + auto setDbCachelast = [&](int8_t cachelast) { expect.cacheLast = cachelast; }; auto setDbCachelastSize = [&](int8_t cachelastSize) { expect.cacheLastSize = cachelastSize; }; - auto setDbCompressionFunc = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; - auto setDbDaysFunc = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; - auto setDbFsyncFunc = [&](int32_t fsyncPeriod) { expect.walFsyncPeriod = fsyncPeriod; }; - auto setDbMaxRowsFunc = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; - auto setDbMinRowsFunc = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; - auto setDbKeepFunc = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { + auto setDbCompression = [&](int8_t compressionLevel) { expect.compression = compressionLevel; }; + auto setDbDays = [&](int32_t daysPerFile) { expect.daysPerFile = daysPerFile; }; + auto setDbFsync = [&](int32_t fsyncPeriod) { expect.walFsyncPeriod = fsyncPeriod; }; + auto setDbMaxRows = [&](int32_t maxRowsPerBlock) { expect.maxRows = maxRowsPerBlock; }; + auto setDbMinRows = [&](int32_t minRowsPerBlock) { expect.minRows = minRowsPerBlock; }; + auto setDbKeep = [&](int32_t keep0, int32_t keep1 = 0, int32_t keep2 = 0) { expect.daysToKeep0 = keep0; expect.daysToKeep1 = 0 == keep1 ? expect.daysToKeep0 : keep1; expect.daysToKeep2 = 0 == keep2 ? expect.daysToKeep1 : keep2; }; - auto setDbPagesFunc = [&](int32_t pages) { expect.pages = pages; }; - auto setDbPageSizeFunc = [&](int32_t pagesize) { expect.pageSize = pagesize; }; - auto setDbPrecisionFunc = [&](int8_t precision) { expect.precision = precision; }; - auto setDbReplicaFunc = [&](int8_t replica) { expect.replications = replica; }; - auto setDbStrictaFunc = [&](int8_t strict) { expect.strict = strict; }; - auto setDbWalLevelFunc = [&](int8_t walLevel) { expect.walLevel = walLevel; }; - auto setDbVgroupsFunc = [&](int32_t numOfVgroups) { expect.numOfVgroups = numOfVgroups; }; - auto setDbSingleStableFunc = [&](int8_t singleStable) { expect.numOfStables = singleStable; }; - auto addDbRetentionFunc = [&](int64_t freq, int64_t keep, int8_t freqUnit, int8_t keepUnit) { + auto setDbPages = [&](int32_t pages) { expect.pages = pages; }; + auto setDbPageSize = [&](int32_t pagesize) { expect.pageSize = pagesize; }; + auto setDbPrecision = [&](int8_t precision) { expect.precision = precision; }; + auto setDbReplica = [&](int8_t replica) { expect.replications = replica; }; + auto setDbStricta = [&](int8_t strict) { expect.strict = strict; }; + auto setDbWalLevel = [&](int8_t walLevel) { expect.walLevel = walLevel; }; + auto setDbVgroups = [&](int32_t numOfVgroups) { expect.numOfVgroups = numOfVgroups; }; + auto setDbSingleStable = [&](int8_t singleStable) { expect.numOfStables = singleStable; }; + auto addDbRetention = [&](int64_t freq, int64_t keep, int8_t freqUnit, int8_t keepUnit) { SRetention retention = {0}; retention.freq = freq; retention.keep = keep; @@ -135,7 +178,7 @@ TEST_F(ParserInitialCTest, createDatabase) { taosArrayPush(expect.pRetensions, &retention); ++expect.numOfRetensions; }; - auto setDbSchemalessFunc = [&](int8_t schemaless) { expect.schemaless = schemaless; }; + auto setDbSchemaless = [&](int8_t schemaless) { expect.schemaless = schemaless; }; auto setDbWalRetentionPeriod = [&](int32_t walRetentionPeriod) { expect.walRetentionPeriod = walRetentionPeriod; }; auto setDbWalRetentionSize = [&](int32_t walRetentionSize) { expect.walRetentionSize = walRetentionSize; }; auto setDbWalRollPeriod = [&](int32_t walRollPeriod) { expect.walRollPeriod = walRollPeriod; }; @@ -147,6 +190,7 @@ TEST_F(ParserInitialCTest, createDatabase) { setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_DATABASE_STMT); + ASSERT_EQ(pQuery->pCmdMsg->msgType, TDMT_MND_CREATE_DB); SCreateDbReq req = {0}; ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateDbReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); @@ -195,32 +239,32 @@ TEST_F(ParserInitialCTest, createDatabase) { tFreeSCreateDbReq(&req); }); - setCreateDbReqFunc("wxy_db"); + setCreateDbReq("wxy_db"); run("CREATE DATABASE wxy_db"); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbBufferFunc(64); - setDbCachelastFunc(2); + setCreateDbReq("wxy_db", 1); + setDbBuffer(64); + setDbCachelast(2); setDbCachelastSize(20); - setDbCompressionFunc(1); - setDbDaysFunc(100 * 1440); - setDbFsyncFunc(100); - setDbMaxRowsFunc(1000); - setDbMinRowsFunc(100); - setDbKeepFunc(1440 * 1440); - setDbPagesFunc(96); - setDbPageSizeFunc(8); - setDbPrecisionFunc(TSDB_TIME_PRECISION_NANO); - setDbReplicaFunc(3); - addDbRetentionFunc(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); - addDbRetentionFunc(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - addDbRetentionFunc(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); - // setDbStrictaFunc(1); - setDbWalLevelFunc(2); - setDbVgroupsFunc(100); - setDbSingleStableFunc(1); - setDbSchemalessFunc(1); + setDbCompression(1); + setDbDays(100 * 1440); + setDbFsync(100); + setDbMaxRows(1000); + setDbMinRows(100); + setDbKeep(1440 * 1440); + setDbPages(96); + setDbPageSize(8); + setDbPrecision(TSDB_TIME_PRECISION_NANO); + setDbReplica(3); + addDbRetention(15 * MILLISECOND_PER_SECOND, 7 * MILLISECOND_PER_DAY, TIME_UNIT_SECOND, TIME_UNIT_DAY); + addDbRetention(1 * MILLISECOND_PER_MINUTE, 21 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); + addDbRetention(15 * MILLISECOND_PER_MINUTE, 500 * MILLISECOND_PER_DAY, TIME_UNIT_MINUTE, TIME_UNIT_DAY); + // setDbStricta(1); + setDbWalLevel(2); + setDbVgroups(100); + setDbSingleStable(1); + setDbSchemaless(1); setDbWalRetentionPeriod(-1); setDbWalRetentionSize(-1); setDbWalRollPeriod(10); @@ -259,16 +303,16 @@ TEST_F(ParserInitialCTest, createDatabase) { "TSDB_PAGESIZE 32"); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbDaysFunc(100); - setDbKeepFunc(1440, 300 * 60, 400 * 1440); + setCreateDbReq("wxy_db", 1); + setDbDays(100); + setDbKeep(1440, 300 * 60, 400 * 1440); run("CREATE DATABASE IF NOT EXISTS wxy_db " "DURATION 100m " "KEEP 1440m,300h,400d "); clearCreateDbReq(); - setCreateDbReqFunc("wxy_db", 1); - setDbReplicaFunc(3); + setCreateDbReq("wxy_db", 1); + setDbReplica(3); setDbWalRetentionPeriod(TSDB_REPS_DEF_DB_WAL_RET_PERIOD); setDbWalRetentionSize(TSDB_REPS_DEF_DB_WAL_RET_SIZE); setDbWalRollPeriod(TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD); @@ -287,6 +331,9 @@ TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) { run("create database db2 retentions 15s:7d,5m:21d,10m:10d", TSDB_CODE_PAR_INVALID_DB_OPTION); } +/* + * CREATE DNODE {dnode_endpoint | dnode_host_name PORT port_val} + */ TEST_F(ParserInitialCTest, createDnode) { useDb("root", "test"); @@ -294,7 +341,7 @@ TEST_F(ParserInitialCTest, createDnode) { auto clearCreateDnodeReq = [&]() { memset(&expect, 0, sizeof(SCreateDnodeReq)); }; - auto setCreateDnodeReqFunc = [&](const char* pFqdn, int32_t port = tsServerPort) { + auto setCreateDnodeReq = [&](const char* pFqdn, int32_t port = tsServerPort) { strcpy(expect.fqdn, pFqdn); expect.port = port; }; @@ -308,39 +355,41 @@ TEST_F(ParserInitialCTest, createDnode) { ASSERT_EQ(req.port, expect.port); }); - setCreateDnodeReqFunc("abc1", 7030); + setCreateDnodeReq("abc1", 7030); run("CREATE DNODE 'abc1' PORT 7030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("1.1.1.1", 8030); + setCreateDnodeReq("1.1.1.1", 8030); run("CREATE DNODE 1.1.1.1 PORT 8030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("host1", 9030); + setCreateDnodeReq("host1", 9030); run("CREATE DNODE host1 PORT 9030"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("abc2", 7040); + setCreateDnodeReq("abc2", 7040); run("CREATE DNODE 'abc2:7040'"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("1.1.1.2"); + setCreateDnodeReq("1.1.1.2"); run("CREATE DNODE 1.1.1.2"); clearCreateDnodeReq(); - setCreateDnodeReqFunc("host2"); + setCreateDnodeReq("host2"); run("CREATE DNODE host2"); clearCreateDnodeReq(); } -// CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +/* + * CREATE [AGGREGATE] FUNCTION [IF NOT EXISTS] func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] + */ TEST_F(ParserInitialCTest, createFunction) { useDb("root", "test"); SCreateFuncReq expect = {0}; - auto setCreateFuncReqFunc = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0, - int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0) { + auto setCreateFuncReq = [&](const char* pUdfName, int8_t outputType, int32_t outputBytes = 0, + int8_t funcType = TSDB_FUNC_TYPE_SCALAR, int8_t igExists = 0, int32_t bufSize = 0) { memset(&expect, 0, sizeof(SCreateFuncReq)); strcpy(expect.name, pUdfName); expect.igExists = igExists; @@ -365,13 +414,69 @@ TEST_F(ParserInitialCTest, createFunction) { ASSERT_EQ(req.bufSize, expect.bufSize); }); - setCreateFuncReqFunc("udf1", TSDB_DATA_TYPE_INT); + setCreateFuncReq("udf1", TSDB_DATA_TYPE_INT); // run("CREATE FUNCTION udf1 AS './build/lib/libudf1.so' OUTPUTTYPE INT"); - setCreateFuncReqFunc("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8); + setCreateFuncReq("udf2", TSDB_DATA_TYPE_DOUBLE, 0, TSDB_FUNC_TYPE_AGGREGATE, 1, 8); // run("CREATE AGGREGATE FUNCTION IF NOT EXISTS udf2 AS './build/lib/libudf2.so' OUTPUTTYPE DOUBLE BUFSIZE 8"); } +/* + * CREATE MNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createMnode) { + useDb("root", "test"); + + SMCreateMnodeReq expect = {0}; + + auto setCreateMnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_MNODE_STMT); + SMCreateMnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setCreateMnodeReq(1); + run("CREATE MNODE ON DNODE 1"); +} + +/* + * CREATE QNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createQnode) { + useDb("root", "test"); + + SMCreateQnodeReq expect = {0}; + + auto setCreateQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_QNODE_STMT); + SMCreateQnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setCreateQnodeReq(1); + run("CREATE QNODE ON DNODE 1"); +} + +/* + * CREATE SMA INDEX index_name ON tb_name index_option + * + * index_option: + * FUNCTION(functions) INTERVAL(interval_val [, interval_offset]) [SLIDING(sliding_val)] + * [WATERMARK(watermark_val)] [MAX_DELAY(max_delay_val)] + * + * functions: + * function [, function] ... + */ TEST_F(ParserInitialCTest, createSmaIndex) { useDb("root", "test"); @@ -439,24 +544,39 @@ TEST_F(ParserInitialCTest, createSmaIndex) { "DELETE_MARK 1000s"); } -TEST_F(ParserInitialCTest, createMnode) { +/* + * CREATE SNODE ON DNODE dnode_id + */ +TEST_F(ParserInitialCTest, createSnode) { useDb("root", "test"); - run("CREATE MNODE ON DNODE 1"); -} + SMCreateSnodeReq expect = {0}; -TEST_F(ParserInitialCTest, createQnode) { - useDb("root", "test"); + auto setCreateSnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; - run("CREATE QNODE ON DNODE 1"); -} + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_SNODE_STMT); + SMCreateSnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); -TEST_F(ParserInitialCTest, createSnode) { - useDb("root", "test"); + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + setCreateSnodeReq(1); run("CREATE SNODE ON DNODE 1"); } +/* + * CREATE STABLE [IF NOT EXISTS] stb_name (create_definition [, create_definitionn] ...) + * TAGS (create_definition [, create_definition] ...) [table_options] + * + * create_definition: + * col_name column_definition + * + * column_definition: + * type_name [COMMENT 'string_value'] + */ TEST_F(ParserInitialCTest, createStable) { useDb("root", "test"); @@ -467,7 +587,7 @@ TEST_F(ParserInitialCTest, createStable) { memset(&expect, 0, sizeof(SMCreateStbReq)); }; - auto setCreateStbReqFunc = + auto setCreateStbReq = [&](const char* pDbName, const char* pTbName, int8_t igExists = 0, int64_t delay1 = -1, int64_t delay2 = -1, int64_t watermark1 = TSDB_DEFAULT_ROLLUP_WATERMARK, int64_t watermark2 = TSDB_DEFAULT_ROLLUP_WATERMARK, int64_t deleteMark1 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, int64_t deleteMark2 = TSDB_DEFAULT_ROLLUP_DELETE_MARK, @@ -483,13 +603,13 @@ TEST_F(ParserInitialCTest, createStable) { expect.deleteMark2 = deleteMark2; // expect.ttl = ttl; if (nullptr != pComment) { - expect.pComment = strdup(pComment); + expect.pComment = taosStrdup(pComment); expect.commentLen = strlen(pComment); } }; - auto addFieldToCreateStbReqFunc = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, - int8_t flags = COL_SMA_ON) { + auto addFieldToCreateStbReq = [&](bool col, const char* pFieldName, uint8_t type, int32_t bytes = 0, + int8_t flags = COL_SMA_ON) { SField field = {0}; strcpy(field.name, pFieldName); field.type = type; @@ -565,46 +685,46 @@ TEST_F(ParserInitialCTest, createStable) { tFreeSMCreateStbReq(&req); }); - setCreateStbReqFunc("test", "t1"); - addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); - addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(false, "id", TSDB_DATA_TYPE_INT); + setCreateStbReq("test", "t1"); + addFieldToCreateStbReq(true, "ts", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReq(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(false, "id", TSDB_DATA_TYPE_INT); run("CREATE STABLE t1(ts TIMESTAMP, c1 INT) TAGS(id INT)"); clearCreateStbReq(); - setCreateStbReqFunc("rollup_db", "t1", 1, 100 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_MINUTE, 10, - 1 * MILLISECOND_PER_MINUTE, 1000 * MILLISECOND_PER_SECOND, 200 * MILLISECOND_PER_MINUTE, 100, - "test create table"); - addFieldToCreateStbReqFunc(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - addFieldToCreateStbReqFunc(true, "c1", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(true, "c2", TSDB_DATA_TYPE_UINT); - addFieldToCreateStbReqFunc(true, "c3", TSDB_DATA_TYPE_BIGINT); - addFieldToCreateStbReqFunc(true, "c4", TSDB_DATA_TYPE_UBIGINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c5", TSDB_DATA_TYPE_FLOAT, 0, 0); - addFieldToCreateStbReqFunc(true, "c6", TSDB_DATA_TYPE_DOUBLE, 0, 0); - addFieldToCreateStbReqFunc(true, "c7", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(true, "c8", TSDB_DATA_TYPE_SMALLINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c9", TSDB_DATA_TYPE_USMALLINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c10", TSDB_DATA_TYPE_TINYINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c11", TSDB_DATA_TYPE_UTINYINT, 0, 0); - addFieldToCreateStbReqFunc(true, "c12", TSDB_DATA_TYPE_BOOL, 0, 0); - addFieldToCreateStbReqFunc(true, "c13", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(true, "c14", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE, 0); - addFieldToCreateStbReqFunc(false, "a1", TSDB_DATA_TYPE_TIMESTAMP); - addFieldToCreateStbReqFunc(false, "a2", TSDB_DATA_TYPE_INT); - addFieldToCreateStbReqFunc(false, "a3", TSDB_DATA_TYPE_UINT); - addFieldToCreateStbReqFunc(false, "a4", TSDB_DATA_TYPE_BIGINT); - addFieldToCreateStbReqFunc(false, "a5", TSDB_DATA_TYPE_UBIGINT); - addFieldToCreateStbReqFunc(false, "a6", TSDB_DATA_TYPE_FLOAT); - addFieldToCreateStbReqFunc(false, "a7", TSDB_DATA_TYPE_DOUBLE); - addFieldToCreateStbReqFunc(false, "a8", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE); - addFieldToCreateStbReqFunc(false, "a9", TSDB_DATA_TYPE_SMALLINT); - addFieldToCreateStbReqFunc(false, "a10", TSDB_DATA_TYPE_USMALLINT); - addFieldToCreateStbReqFunc(false, "a11", TSDB_DATA_TYPE_TINYINT); - addFieldToCreateStbReqFunc(false, "a12", TSDB_DATA_TYPE_UTINYINT); - addFieldToCreateStbReqFunc(false, "a13", TSDB_DATA_TYPE_BOOL); - addFieldToCreateStbReqFunc(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - addFieldToCreateStbReqFunc(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); + setCreateStbReq("rollup_db", "t1", 1, 100 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_MINUTE, 10, + 1 * MILLISECOND_PER_MINUTE, 1000 * MILLISECOND_PER_SECOND, 200 * MILLISECOND_PER_MINUTE, 100, + "test create table"); + addFieldToCreateStbReq(true, "ts", TSDB_DATA_TYPE_TIMESTAMP, 0, 0); + addFieldToCreateStbReq(true, "c1", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(true, "c2", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReq(true, "c3", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReq(true, "c4", TSDB_DATA_TYPE_UBIGINT, 0, 0); + addFieldToCreateStbReq(true, "c5", TSDB_DATA_TYPE_FLOAT, 0, 0); + addFieldToCreateStbReq(true, "c6", TSDB_DATA_TYPE_DOUBLE, 0, 0); + addFieldToCreateStbReq(true, "c7", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(true, "c8", TSDB_DATA_TYPE_SMALLINT, 0, 0); + addFieldToCreateStbReq(true, "c9", TSDB_DATA_TYPE_USMALLINT, 0, 0); + addFieldToCreateStbReq(true, "c10", TSDB_DATA_TYPE_TINYINT, 0, 0); + addFieldToCreateStbReq(true, "c11", TSDB_DATA_TYPE_UTINYINT, 0, 0); + addFieldToCreateStbReq(true, "c12", TSDB_DATA_TYPE_BOOL, 0, 0); + addFieldToCreateStbReq(true, "c13", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(true, "c14", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE, 0); + addFieldToCreateStbReq(false, "a1", TSDB_DATA_TYPE_TIMESTAMP); + addFieldToCreateStbReq(false, "a2", TSDB_DATA_TYPE_INT); + addFieldToCreateStbReq(false, "a3", TSDB_DATA_TYPE_UINT); + addFieldToCreateStbReq(false, "a4", TSDB_DATA_TYPE_BIGINT); + addFieldToCreateStbReq(false, "a5", TSDB_DATA_TYPE_UBIGINT); + addFieldToCreateStbReq(false, "a6", TSDB_DATA_TYPE_FLOAT); + addFieldToCreateStbReq(false, "a7", TSDB_DATA_TYPE_DOUBLE); + addFieldToCreateStbReq(false, "a8", TSDB_DATA_TYPE_BINARY, 20 + VARSTR_HEADER_SIZE); + addFieldToCreateStbReq(false, "a9", TSDB_DATA_TYPE_SMALLINT); + addFieldToCreateStbReq(false, "a10", TSDB_DATA_TYPE_USMALLINT); + addFieldToCreateStbReq(false, "a11", TSDB_DATA_TYPE_TINYINT); + addFieldToCreateStbReq(false, "a12", TSDB_DATA_TYPE_UTINYINT); + addFieldToCreateStbReq(false, "a13", TSDB_DATA_TYPE_BOOL); + addFieldToCreateStbReq(false, "a14", TSDB_DATA_TYPE_NCHAR, 30 * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " @@ -630,6 +750,20 @@ TEST_F(ParserInitialCTest, createStableSemanticCheck) { TSDB_CODE_PAR_INVALID_TABLE_OPTION); } +/* + * CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] + * INTO stb_name [TAGS (create_definition [, create_definition] ...)] [SUBTABLE (expr)] AS subquery + * + * stream_options: + * stream_option ... + * + * stream_option: { + * TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + * | WATERMARK time + * | IGNORE EXPIRED value + * | FILL_HISTORY value + * } + */ TEST_F(ParserInitialCTest, createStream) { useDb("root", "test"); @@ -641,23 +775,33 @@ TEST_F(ParserInitialCTest, createStream) { }; auto setCreateStreamReq = [&](const char* pStream, const char* pSrcDb, const char* pSql, const char* pDstStb, - int8_t igExists = 0, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, int64_t maxDelay = 0, - int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, - int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY, - int8_t igUpdate = STREAM_DEFAULT_IGNORE_UPDATE) { + int8_t igExists = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pStream); snprintf(expect.sourceDB, sizeof(expect.sourceDB), "0.%s", pSrcDb); snprintf(expect.targetStbFullName, sizeof(expect.targetStbFullName), "0.test.%s", pDstStb); expect.igExists = igExists; - expect.sql = strdup(pSql); - expect.triggerType = triggerType; - expect.maxDelay = maxDelay; - expect.watermark = watermark; - expect.fillHistory = fillHistory; - expect.igExpired = igExpired; - expect.igUpdate = igUpdate; + expect.sql = taosStrdup(pSql); + expect.createStb = STREAM_CREATE_STABLE_TRUE; + expect.triggerType = STREAM_TRIGGER_AT_ONCE; + expect.maxDelay = 0; + expect.watermark = 0; + expect.fillHistory = STREAM_DEFAULT_FILL_HISTORY; + expect.igExpired = STREAM_DEFAULT_IGNORE_EXPIRED; }; + auto setStreamOptions = + [&](int8_t createStb = STREAM_CREATE_STABLE_TRUE, int8_t triggerType = STREAM_TRIGGER_AT_ONCE, + int64_t maxDelay = 0, int64_t watermark = 0, int8_t igExpired = STREAM_DEFAULT_IGNORE_EXPIRED, + int8_t fillHistory = STREAM_DEFAULT_FILL_HISTORY, int8_t igUpdate = STREAM_DEFAULT_IGNORE_UPDATE) { + expect.createStb = createStb; + expect.triggerType = triggerType; + expect.maxDelay = maxDelay; + expect.watermark = watermark; + expect.fillHistory = fillHistory; + expect.igExpired = igExpired; + expect.igUpdate = igUpdate; + }; + auto addTag = [&](const char* pFieldName, uint8_t type, int32_t bytes = 0) { SField field = {0}; strcpy(field.name, pFieldName); @@ -701,21 +845,25 @@ TEST_F(ParserInitialCTest, createStream) { ASSERT_EQ(pField->flags, pExpectField->flags); } } + ASSERT_EQ(req.checkpointFreq, expect.checkpointFreq); + ASSERT_EQ(req.createStb, expect.createStb); ASSERT_EQ(req.igUpdate, expect.igUpdate); tFreeSCMCreateStreamReq(&req); }); - setCreateStreamReq("s1", "test", "create stream s1 into st1 as select count(*) from t1 interval(10s)", "st1"); - run("CREATE STREAM s1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); + setCreateStreamReq("s1", "test", "create stream s1 into st3 as select count(*) from t1 interval(10s)", "st3"); + run("CREATE STREAM s1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); setCreateStreamReq( "s1", "test", "create stream if not exists s1 trigger max_delay 20s watermark 10s ignore expired 0 fill_history 1 ignore " - "update 1 into st1 as select count(*) from t1 interval(10s)", - "st1", 1, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, 10 * MILLISECOND_PER_SECOND, 0, 1, 1); + "update 1 into st3 as select count(*) from t1 interval(10s)", + "st3", 1); + setStreamOptions(STREAM_CREATE_STABLE_TRUE, STREAM_TRIGGER_MAX_DELAY, 20 * MILLISECOND_PER_SECOND, + 10 * MILLISECOND_PER_SECOND, 0, 1, 1); run("CREATE STREAM IF NOT EXISTS s1 TRIGGER MAX_DELAY 20s WATERMARK 10s IGNORE EXPIRED 0 FILL_HISTORY 1 IGNORE " - "UPDATE 1 INTO st1 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); + "UPDATE 1 INTO st3 AS SELECT COUNT(*) FROM t1 INTERVAL(10S)"); clearCreateStreamReq(); setCreateStreamReq("s1", "test", @@ -727,6 +875,15 @@ TEST_F(ParserInitialCTest, createStream) { run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " "AS SELECT _WSTART wstart, COUNT(*) cnt FROM st1 PARTITION BY TBNAME tname, tag1 id INTERVAL(10S)"); clearCreateStreamReq(); + + // st1 already exists + setCreateStreamReq( + "s1", "test", + "create stream s1 into st1 tags(tag2) as select max(c1), c2 from t1 partition by tbname tag2 interval(10s)", + "st1"); + setStreamOptions(STREAM_CREATE_STABLE_FALSE); + run("CREATE STREAM s1 INTO st1 TAGS(tag2) AS SELECT MAX(c1), c2 FROM t1 PARTITION BY TBNAME tag2 INTERVAL(10S)"); + clearCreateStreamReq(); } TEST_F(ParserInitialCTest, createStreamSemanticCheck) { @@ -736,9 +893,103 @@ TEST_F(ParserInitialCTest, createStreamSemanticCheck) { TSDB_CODE_PAR_STREAM_NOT_ALLOWED_FUNC); } +/* + * CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) [table_options] + * + * CREATE TABLE create_subtable_clause + * + * CREATE TABLE [IF NOT EXISTS] [db_name.]tb_name (create_definition [, create_definitionn] ...) + * [TAGS (create_definition [, create_definitionn] ...)] + * [table_options] + * + * create_subtable_clause: { + * create_subtable_clause [create_subtable_clause] ... + * | [IF NOT EXISTS] [db_name.]tb_name USING [db_name.]stb_name [(tag_name [, tag_name] ...)] + * TAGS (tag_value [, tag_value] ...) + * } + * + * create_definition: + * col_name column_definition + * + * column_definition: + * type_name [comment 'string_value'] + * + * table_options: + * table_option ... + * + * table_option: { + * COMMENT 'string_value' + * | WATERMARK duration[,duration] + * | MAX_DELAY duration[,duration] + * | ROLLUP(func_name [, func_name] ...) + * | SMA(col_name [, col_name] ...) + * | TTL value + * } + */ TEST_F(ParserInitialCTest, createTable) { useDb("root", "test"); + SVCreateTbBatchReq expect = {0}; + + auto addCreateTbReq = [&](const char* pName, bool ignoreExists = false, int32_t ttl = TSDB_DEFAULT_TABLE_TTL, + const char* pComment = nullptr) { + SVCreateTbReq req = {0}; + req.name = taosStrdup(pName); + if (ignoreExists) { + req.flags |= TD_CREATE_IF_NOT_EXISTS; + } + req.ttl = ttl; + if (nullptr != pComment) { + req.comment = taosStrdup(pComment); + req.commentLen = strlen(pComment); + } + ++expect.nReqs; + if (nullptr == expect.pArray) { + expect.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVCreateTbReq)); + } + taosArrayPush(expect.pArray, &req); + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + return; // todo + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_VNODE_MODIFY_STMT); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; + + ASSERT_EQ(pStmt->sqlNodeType, QUERY_NODE_CREATE_TABLE_STMT); + ASSERT_NE(pStmt->pDataBlocks, nullptr); + int32_t numOfBlocks = taosArrayGetSize(pStmt->pDataBlocks); + for (int32_t i = 0; i < numOfBlocks; ++i) { + SVgDataBlocks* pVgData = (SVgDataBlocks*)taosArrayGetP(pStmt->pDataBlocks, i); + void* pBuf = POINTER_SHIFT(pVgData->pData, sizeof(SMsgHead)); + SVCreateTbBatchReq req = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, (uint8_t*)pBuf, pVgData->size); + ASSERT_EQ(tDecodeSVCreateTbBatchReq(&coder, &req), TSDB_CODE_SUCCESS); + ASSERT_EQ(req.nReqs, expect.nReqs); + for (int32_t j = 0; j < req.nReqs; ++j) { + SVCreateTbReq* pReq = req.pReqs + j; + SVCreateTbReq* pExpect = (SVCreateTbReq*)taosArrayGet(expect.pArray, j); + ASSERT_EQ(pReq->flags, pExpect->flags); + ASSERT_EQ(std::string(pReq->name), std::string(pExpect->name)); + ASSERT_EQ(pReq->uid, pExpect->uid); + ASSERT_EQ(pReq->ctime, pExpect->ctime); + ASSERT_EQ(pReq->ttl, pExpect->ttl); + ASSERT_EQ(pReq->commentLen, pExpect->commentLen); + ASSERT_EQ(std::string(pReq->comment), std::string(pExpect->comment)); + ASSERT_EQ(pReq->type, pExpect->type); + if (TD_NORMAL_TABLE == pExpect->type) { + ASSERT_EQ(pReq->ntb.schemaRow.version, pExpect->ntb.schemaRow.version); + ASSERT_EQ(pReq->ntb.schemaRow.nCols, pExpect->ntb.schemaRow.nCols); + } else if (TD_CHILD_TABLE == pExpect->type) { + ASSERT_EQ(std::string(pReq->ctb.stbName), std::string(pExpect->ctb.stbName)); + ASSERT_EQ(pReq->ctb.tagNum, pExpect->ctb.tagNum); + ASSERT_EQ(pReq->ctb.suid, pExpect->ctb.suid); + } + } + tDecoderClear(&coder); + } + }); + run("CREATE TABLE t1(ts TIMESTAMP, c1 INT)"); run("CREATE TABLE IF NOT EXISTS test.t1(" @@ -763,7 +1014,7 @@ TEST_F(ParserInitialCTest, createTable) { "IF NOT EXISTS test.t2 USING test.st1 (tag1, tag2) TAGS(2, 'abc') " "IF NOT EXISTS test.t3 USING test.st1 (tag1, tag2) TAGS(3, 'abc') "); - // run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW + 1S)"); + run("CREATE TABLE IF NOT EXISTS t1 USING st1 TAGS(1, 'wxy', NOW + 1S)"); } TEST_F(ParserInitialCTest, createTableSemanticCheck) { @@ -781,6 +1032,11 @@ TEST_F(ParserInitialCTest, createTableSemanticCheck) { run(sql, TSDB_CODE_PAR_TOO_MANY_COLUMNS); } +/* + * CREATE TOPIC [IF NOT EXISTS] topic_name AS subquery + * + * CREATE TOPIC [IF NOT EXISTS] topic_name [WITH META] AS {DATABASE db_name | STABLE stb_name } + */ TEST_F(ParserInitialCTest, createTopic) { useDb("root", "test"); @@ -788,8 +1044,8 @@ TEST_F(ParserInitialCTest, createTopic) { auto clearCreateTopicReq = [&]() { memset(&expect, 0, sizeof(SCMCreateTopicReq)); }; - auto setCreateTopicReqFunc = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst, - const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) { + auto setCreateTopicReq = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst, + const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pTopicName); expect.igExists = igExists; expect.sql = (char*)pSql; @@ -833,31 +1089,34 @@ TEST_F(ParserInitialCTest, createTopic) { tFreeSCMCreateTopicReq(&req); }); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 as select * from t1", "ast"); + setCreateTopicReq("tp1", 0, "create topic tp1 as select * from t1", "ast"); run("CREATE TOPIC tp1 AS SELECT * FROM t1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as select ts, ceil(c1) from t1", "ast"); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as select ts, ceil(c1) from t1", "ast"); run("CREATE TOPIC IF NOT EXISTS tp1 AS SELECT ts, CEIL(c1) FROM t1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 as database test", nullptr, "test"); + setCreateTopicReq("tp1", 0, "create topic tp1 as database test", nullptr, "test"); run("CREATE TOPIC tp1 AS DATABASE test"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1); + setCreateTopicReq("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1); run("CREATE TOPIC tp1 WITH META AS DATABASE test"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1"); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1"); run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1"); clearCreateTopicReq(); - setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); + setCreateTopicReq("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); clearCreateTopicReq(); } +/* + * CREATE USER use_name PASS password [SYSINFO value] + */ TEST_F(ParserInitialCTest, createUser) { useDb("root", "test"); diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 3ded71242b961ceaf12abbdff44272c7fa6cb32c..203156ec883427f943c847a24ea2ac734c645f2a 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -99,7 +99,7 @@ TEST_F(ParserInitialDTest, dropDnode) { expect.force = force; }; - auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port, bool force = false) { + auto setDropDnodeReqByEndpoint = [&](const char* pFqdn, int32_t port = tsServerPort, bool force = false) { strcpy(expect.fqdn, pFqdn); expect.port = port; expect.force = force; @@ -131,6 +131,14 @@ TEST_F(ParserInitialDTest, dropDnode) { setDropDnodeReqByEndpoint("host2", 8030, true); run("DROP DNODE 'host2:8030' FORCE"); clearDropDnodeReq(); + + setDropDnodeReqByEndpoint("host1"); + run("DROP DNODE host1"); + clearDropDnodeReq(); + + setDropDnodeReqByEndpoint("host2", tsServerPort, true); + run("DROP DNODE host2 FORCE"); + clearDropDnodeReq(); } // todo DROP function @@ -174,7 +182,21 @@ TEST_F(ParserInitialDTest, dropMnode) { TEST_F(ParserInitialDTest, dropQnode) { useDb("root", "test"); - run("DROP qnode on dnode 1"); + SMDropQnodeReq expect = {0}; + + auto setDropQnodeReq = [&](int32_t dnodeId) { expect.dnodeId = dnodeId; }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_QNODE_STMT); + SMDropQnodeReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == + tDeserializeSCreateDropMQSNodeReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.dnodeId, expect.dnodeId); + }); + + setDropQnodeReq(1); + run("DROP QNODE ON DNODE 1"); } TEST_F(ParserInitialDTest, dropSnode) { @@ -223,6 +245,7 @@ TEST_F(ParserInitialDTest, dropTable) { useDb("root", "test"); run("DROP TABLE t1"); + run("DROP TABLE t1, st1s1, st1s2"); } TEST_F(ParserInitialDTest, dropTopic) { @@ -237,7 +260,20 @@ TEST_F(ParserInitialDTest, dropUser) { login("root"); useDb("root", "test"); - run("DROP user wxy"); + SDropUserReq expect = {0}; + + auto setDropUserReq = [&](const char* pUser) { sprintf(expect.user, "%s", pUser); }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_USER_STMT); + SDropUserReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSDropUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.user), std::string(expect.user)); + }); + + setDropUserReq("wxy"); + run("DROP USER wxy"); } } // namespace ParserTest diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index de2ce554593749ff420cc13312c84414d0e99f73..8d13d7cf0e75bd5672ed0be1dc096fbb7a4868ea 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -86,6 +86,7 @@ static void parseArg(int argc, char* argv[]) { {"dump", no_argument, NULL, 'd'}, {"async", required_argument, NULL, 'a'}, {"skipSql", required_argument, NULL, 's'}, + {"limitSql", required_argument, NULL, 'i'}, {"log", required_argument, NULL, 'l'}, {0, 0, 0, 0} }; @@ -101,6 +102,9 @@ static void parseArg(int argc, char* argv[]) { case 's': setSkipSqlNum(optarg); break; + case 'i': + setLimitSqlNum(optarg); + break; case 'l': setLogLevel(optarg); break; diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index dbae302e426fb0c842c175ae17b71a1f1767e65d..f18dd4f17f67b9d35990d6c326b11b26c3033a47 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -49,9 +49,11 @@ bool g_dump = false; bool g_testAsyncApis = true; int32_t g_logLevel = 131; int32_t g_skipSql = 0; +int32_t g_limitSql = 0; -void setAsyncFlag(const char* pFlag) { g_testAsyncApis = stoi(pFlag) > 0 ? true : false; } -void setSkipSqlNum(const char* pNum) { g_skipSql = stoi(pNum); } +void setAsyncFlag(const char* pArg) { g_testAsyncApis = stoi(pArg) > 0 ? true : false; } +void setSkipSqlNum(const char* pArg) { g_skipSql = stoi(pArg); } +void setLimitSqlNum(const char* pArg) { g_limitSql = stoi(pArg); } struct TerminateFlag : public exception { const char* what() const throw() { return "success and terminate"; } @@ -63,22 +65,27 @@ int32_t getLogLevel() { return g_logLevel; } class ParserTestBaseImpl { public: - ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase), sqlNo_(0) {} + ParserTestBaseImpl(ParserTestBase* pBase) : pBase_(pBase), sqlNo_(0), sqlNum_(0) {} void login(const std::string& user) { caseEnv_.user_ = user; } void useDb(const string& acctId, const string& db) { caseEnv_.acctId_ = acctId; caseEnv_.db_ = db; - caseEnv_.nsql_ = g_skipSql; + caseEnv_.numOfSkipSql_ = g_skipSql; + caseEnv_.numOfLimitSql_ = g_limitSql; } void run(const string& sql, int32_t expect, ParserStage checkStage) { ++sqlNo_; - if (caseEnv_.nsql_ > 0) { - --(caseEnv_.nsql_); + if (caseEnv_.numOfSkipSql_ > 0) { + --(caseEnv_.numOfSkipSql_); return; } + if (caseEnv_.numOfLimitSql_ > 0 && caseEnv_.numOfLimitSql_ == sqlNum_) { + return; + } + ++sqlNum_; runInternalFuncs(sql, expect, checkStage); runApis(sql, expect, checkStage); @@ -94,9 +101,10 @@ class ParserTestBaseImpl { string acctId_; string user_; string db_; - int32_t nsql_; + int32_t numOfSkipSql_; + int32_t numOfLimitSql_; - caseEnv() : user_("wangxiaoyu"), nsql_(0) {} + caseEnv() : user_("wangxiaoyu"), numOfSkipSql_(0) {} }; struct stmtEnv { @@ -532,6 +540,7 @@ class ParserTestBaseImpl { stmtRes res_; ParserTestBase* pBase_; int32_t sqlNo_; + int32_t sqlNum_; }; ParserTestBase::ParserTestBase() : impl_(new ParserTestBaseImpl(this)) {} diff --git a/source/libs/parser/test/parTestUtil.h b/source/libs/parser/test/parTestUtil.h index 3e5fab592729a9d98010900be1ef6dbbedc0d07f..5d9680c477d90e0268e3383ff089013006735a8d 100644 --- a/source/libs/parser/test/parTestUtil.h +++ b/source/libs/parser/test/parTestUtil.h @@ -65,10 +65,11 @@ class ParserDdlTest : public ParserTestBase { extern bool g_dump; -extern void setAsyncFlag(const char* pFlag); -extern void setLogLevel(const char* pLogLevel); +extern void setAsyncFlag(const char* pArg); +extern void setLogLevel(const char* pArg); extern int32_t getLogLevel(); -extern void setSkipSqlNum(const char* pNum); +extern void setSkipSqlNum(const char* pArg); +extern void setLimitSqlNum(const char* pArg); } // namespace ParserTest diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index bfde50761ae3995a56af5719c2953eb90cb48d74..c9ee83a64757b6fe951d93fd9c9d917ba357f58b 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -381,6 +381,20 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = addDefaultScanCol(pRealTable->pMeta, &pScan->pScanCols); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pTags && NULL == pSelect->pPartitionByList) { + pScan->pTags = nodesCloneList(pSelect->pTags); + if (NULL == pScan->pTags) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pSubtable && NULL == pSelect->pPartitionByList) { + pScan->pSubtable = nodesCloneNode(pSelect->pSubtable); + if (NULL == pScan->pSubtable) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + // set output if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pScan->pScanCols, &pScan->node.pTargets); @@ -823,6 +837,29 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } +static int32_t createWindowLogicNodeByEvent(SLogicPlanContext* pCxt, SEventWindowNode* pEvent, SSelectStmt* pSelect, + SLogicNode** pLogicNode) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_WINDOW); + if (NULL == pWindow) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pWindow->winType = WINDOW_TYPE_EVENT; + pWindow->node.groupAction = getGroupAction(pCxt, pSelect); + pWindow->node.requireDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_IN_BLOCK : getRequireDataOrder(true, pSelect); + pWindow->node.resultDataOrder = + pCxt->pPlanCxt->streamQuery ? DATA_ORDER_LEVEL_GLOBAL : pWindow->node.requireDataOrder; + pWindow->pStartCond = nodesCloneNode(pEvent->pStartCond); + pWindow->pEndCond = nodesCloneNode(pEvent->pEndCond); + pWindow->pTspk = nodesCloneNode(pEvent->pCol); + if (NULL == pWindow->pStartCond || NULL == pWindow->pEndCond || NULL == pWindow->pTspk) { + nodesDestroyNode((SNode*)pWindow); + return TSDB_CODE_OUT_OF_MEMORY; + } + return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); +} + static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; @@ -835,6 +872,8 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return createWindowLogicNodeBySession(pCxt, (SSessionWindowNode*)pSelect->pWindow, pSelect, pLogicNode); case QUERY_NODE_INTERVAL_WINDOW: return createWindowLogicNodeByInterval(pCxt, (SIntervalWindowNode*)pSelect->pWindow, pSelect, pLogicNode); + case QUERY_NODE_EVENT_WINDOW: + return createWindowLogicNodeByEvent(pCxt, (SEventWindowNode*)pSelect->pWindow, pSelect, pLogicNode); default: break; } @@ -1371,7 +1410,7 @@ static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* static int32_t getMsgType(ENodeType sqlType) { switch (sqlType) { case QUERY_NODE_CREATE_TABLE_STMT: - case QUERY_NODE_CREATE_MULTI_TABLE_STMT: + case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return TDMT_VND_CREATE_TABLE; case QUERY_NODE_DROP_TABLE_STMT: return TDMT_VND_DROP_TABLE; @@ -1385,7 +1424,7 @@ static int32_t getMsgType(ENodeType sqlType) { return TDMT_VND_SUBMIT; } -static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifOpStmt* pStmt, SLogicNode** pLogicNode) { +static int32_t createVnodeModifLogicNode(SLogicPlanContext* pCxt, SVnodeModifyOpStmt* pStmt, SLogicNode** pLogicNode) { SVnodeModifyLogicNode* pModif = (SVnodeModifyLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_VNODE_MODIFY); if (NULL == pModif) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1569,8 +1608,8 @@ static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogi switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: return createSelectLogicNode(pCxt, (SSelectStmt*)pStmt, pLogicNode); - case QUERY_NODE_VNODE_MODIF_STMT: - return createVnodeModifLogicNode(pCxt, (SVnodeModifOpStmt*)pStmt, pLogicNode); + case QUERY_NODE_VNODE_MODIFY_STMT: + return createVnodeModifLogicNode(pCxt, (SVnodeModifyOpStmt*)pStmt, pLogicNode); case QUERY_NODE_EXPLAIN_STMT: return createQueryLogicNode(pCxt, ((SExplainStmt*)pStmt)->pQuery, pLogicNode); case QUERY_NODE_SET_OPERATOR: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 90a72610745fa5d42dbd61cc9868d7776bbb262b..16a5ef7bae1e7aeaa4c0521022bd96913b9c222a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1280,18 +1280,14 @@ static int32_t smaIndexOptFindSmaFunc(SNode* pQueryFunc, SNodeList* pSmaFuncs) { return -1; } -static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, SNodeList** pOutput, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNodeList* pSmaFuncs, + SNodeList** pOutput) { SNodeList* pCols = NULL; SNode* pFunc = NULL; int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; int32_t smaFuncIndex = -1; - *pWStrartIndex = -1; FOREACH(pFunc, pFuncs) { - if (FUNCTION_TYPE_WSTART == ((SFunctionNode*)pFunc)->funcType) { - *pWStrartIndex = index; - } smaFuncIndex = smaIndexOptFindSmaFunc(pFunc, pSmaFuncs); if (smaFuncIndex < 0) { break; @@ -1313,8 +1309,7 @@ static int32_t smaIndexOptCreateSmaCols(SNodeList* pFuncs, uint64_t tableId, SNo return code; } -static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols, - int32_t* pWStrartIndex) { +static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList** pCols) { SWindowLogicNode* pWindow = (SWindowLogicNode*)pScan->node.pParent; if (!smaIndexOptEqualInterval(pScan, pWindow, pIndex)) { return TSDB_CODE_SUCCESS; @@ -1322,36 +1317,34 @@ static int32_t smaIndexOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo SNodeList* pSmaFuncs = NULL; int32_t code = nodesStringToList(pIndex->expr, &pSmaFuncs); if (TSDB_CODE_SUCCESS == code) { - code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols, pWStrartIndex); + code = smaIndexOptCreateSmaCols(pWindow->pFuncs, pIndex->dstTbUid, pSmaFuncs, pCols); } nodesDestroyList(pSmaFuncs); return code; } static int32_t smaIndexOptApplyIndex(SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan, STableIndexInfo* pIndex, - SNodeList* pSmaCols, int32_t wstrartIndex) { + SNodeList* pSmaCols) { SLogicNode* pSmaScan = NULL; int32_t code = smaIndexOptCreateSmaScan(pScan, pIndex, pSmaCols, &pSmaScan); if (TSDB_CODE_SUCCESS == code) { code = replaceLogicNode(pLogicSubplan, pScan->node.pParent, pSmaScan); } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode((SNode*)pScan->node.pParent); + } return code; } -static void smaIndexOptDestroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } - static int32_t smaIndexOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SScanLogicNode* pScan) { int32_t code = TSDB_CODE_SUCCESS; int32_t nindexes = taosArrayGetSize(pScan->pSmaIndexes); for (int32_t i = 0; i < nindexes; ++i) { STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); SNodeList* pSmaCols = NULL; - int32_t wstrartIndex = -1; - code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols, &wstrartIndex); + code = smaIndexOptCouldApplyIndex(pScan, pIndex, &pSmaCols); if (TSDB_CODE_SUCCESS == code && NULL != pSmaCols) { - code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols, wstrartIndex); - taosArrayDestroyEx(pScan->pSmaIndexes, smaIndexOptDestroySmaIndex); - pScan->pSmaIndexes = NULL; + code = smaIndexOptApplyIndex(pLogicSubplan, pScan, pIndex, pSmaCols); pCxt->optimized = true; break; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index f016ca165b8c26f851a27d68e24f27004c21ac61..e9a2efaac79fe237c8e4e93a91cf79f397d27f33 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -610,7 +610,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->accountId = pCxt->pPlanCxt->acctId; pScan->sysInfo = pCxt->pPlanCxt->sysInfo; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_TAGS) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_COLS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { pSubplan->execNode.nodeId = MNODE_HANDLE; @@ -1299,6 +1300,33 @@ static int32_t createStateWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pC return code; } +static int32_t createEventWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, + SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { + SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pWindowLogicNode, + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT : QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT)); + if (NULL == pEvent) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); + int32_t code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pStartCond, &pEvent->pStartCond); + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pWindowLogicNode->pEndCond, &pEvent->pEndCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = createWindowPhysiNodeFinalize(pCxt, pChildren, &pEvent->window, pWindowLogicNode); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pEvent; + } else { + nodesDestroyNode((SNode*)pEvent); + } + + return code; +} + static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { switch (pWindowLogicNode->winType) { @@ -1308,6 +1336,8 @@ static int32_t createWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildr return createSessionWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); case WINDOW_TYPE_STATE: return createStateWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); + case WINDOW_TYPE_EVENT: + return createEventWindowPhysiNode(pCxt, pChildren, pWindowLogicNode, pPhyNode); default: break; } @@ -1689,6 +1719,7 @@ static int32_t createQueryInserter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNod strcpy(pInserter->tableName, pModify->tableName); pInserter->vgId = pModify->pVgroupList->vgroups[0].vgId; pInserter->epSet = pModify->pVgroupList->vgroups[0].epSet; + pInserter->explain = (QUERY_NODE_EXPLAIN_STMT == nodeType(pCxt->pPlanCxt->pAstRoot) ? true : false); vgroupInfoToNodeAddr(pModify->pVgroupList->vgroups, &pSubplan->execNode); int32_t code = setListSlotId(pCxt, pSubplan->pNode->pOutputDataBlockDesc->dataBlockId, -1, pModify->pInsertCols, diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4c8b996a75f19920d8f64bbd91b5d98fbe192f97..361cf33d58bffb63523156d6fab555b8ccb5d5d4 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -742,6 +742,18 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { } } +static int32_t stbSplSplitEventForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t stbSplSplitEvent(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitEventForStream(pCxt, pInfo); + } else { + return stbSplSplitSessionOrStateForBatch(pCxt, pInfo); + } +} + static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } @@ -754,6 +766,8 @@ static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitI return stbSplSplitSession(pCxt, pInfo); case WINDOW_TYPE_STATE: return stbSplSplitState(pCxt, pInfo); + case WINDOW_TYPE_EVENT: + return stbSplSplitEvent(pCxt, pInfo); default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index fd6cb730f8e7c9109e089b06aaf20cd8fd88ca28..4b8009347d5e44ce236f99652b4814e36b8af459 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -200,6 +200,15 @@ static int32_t adjustStateDataRequirement(SWindowLogicNode* pWindow, EDataOrderL return TSDB_CODE_SUCCESS; } +static int32_t adjustEventDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { + if (requirement <= pWindow->node.resultDataOrder) { + return TSDB_CODE_SUCCESS; + } + pWindow->node.resultDataOrder = requirement; + pWindow->node.requireDataOrder = requirement; + return TSDB_CODE_SUCCESS; +} + static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrderLevel requirement) { switch (pWindow->winType) { case WINDOW_TYPE_INTERVAL: @@ -208,6 +217,8 @@ static int32_t adjustWindowDataRequirement(SWindowLogicNode* pWindow, EDataOrder return adjustSessionDataRequirement(pWindow, requirement); case WINDOW_TYPE_STATE: return adjustStateDataRequirement(pWindow, requirement); + case WINDOW_TYPE_EVENT: + return adjustEventDataRequirement(pWindow, requirement); default: break; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 150df76416d71514fc226f35b48ec4f57701d31b..10ab71ab0e68e2a31c798c46e3c2f685a70d2201 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -104,6 +104,8 @@ TEST_F(PlanBasicTest, interpFunc) { run("SELECT _IROWTS, INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); + run("SELECT _IROWTS, INTERP(c1), _ISFILLED FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); + run("SELECT TBNAME, _IROWTS, INTERP(c1) FROM t1 PARTITION BY TBNAME " "RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); } diff --git a/source/libs/cache/inc/cacheDef.h b/source/libs/planner/test/planEventTest.cpp similarity index 60% rename from source/libs/cache/inc/cacheDef.h rename to source/libs/planner/test/planEventTest.cpp index 2e0dbfcdb6065ee7cdb15c479e7755dee9d4a77d..c4db1459982de3c8daf6b8b37795a806ddd77d01 100644 --- a/source/libs/cache/inc/cacheDef.h +++ b/source/libs/planner/test/planEventTest.cpp @@ -13,15 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_CACHE_DEF_H_ -#define _TD_CACHE_DEF_H_ +#include "planTestUtil.h" +#include "planner.h" -#ifdef __cplusplus -extern "C" { -#endif +using namespace std; -#ifdef __cplusplus +class PlanEventTest : public PlannerTestBase {}; + +TEST_F(PlanEventTest, basic) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM t1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); } -#endif -#endif /*_TD_CACHE_DEF_H_*/ \ No newline at end of file +TEST_F(PlanEventTest, stable) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM st1 EVENT_WINDOW START WITH c1 > 10 END WITH c2 = 'abc'"); +} diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 4741d241b5ac8b5541b6f92d08dbe1dd26151099..3a12d62340f1744db5d351d7494e2e181b512d76 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -30,7 +30,7 @@ TEST_F(PlanOtherTest, createTopic) { TEST_F(PlanOtherTest, createStream) { useDb("root", "test"); - run("create stream if not exists s1 trigger window_close watermark 10s into st1 as select count(*) from t1 " + run("create stream if not exists s1 trigger window_close watermark 10s into st3 as select count(*) from t1 " "interval(10s)"); run("CREATE STREAM s1 INTO st3 TAGS(tname VARCHAR(10), id INT) SUBTABLE(CONCAT('new-', tname)) " @@ -43,9 +43,9 @@ TEST_F(PlanOtherTest, createStream) { TEST_F(PlanOtherTest, createStreamUseSTable) { useDb("root", "test"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); - run("CREATE STREAM IF NOT EXISTS s1 into st1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 into st3 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } TEST_F(PlanOtherTest, createSmaIndex) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c74882cf23dac901eab09b779afd56cbfb55dcaa..c68a08682c57886b407dcb9f6ab7c6245ddf61a2 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -116,8 +116,6 @@ int32_t cleanupTaskQueue() { } static void execHelper(struct SSchedMsg* pSchedMsg) { - assert(pSchedMsg != NULL && pSchedMsg->ahandle != NULL); - __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); if (code != 0 && pSchedMsg->msg != NULL) { @@ -126,8 +124,6 @@ static void execHelper(struct SSchedMsg* pSchedMsg) { } int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) { - assert(execFn != NULL); - SSchedMsg schedMsg = {0}; schedMsg.fp = execHelper; schedMsg.ahandle = execFn; @@ -138,7 +134,10 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) } void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { - assert(pMsgBody != NULL); + if (NULL == pMsgBody) { + return; + } + taosMemoryFreeClear(pMsgBody->target.dbFName); taosMemoryFreeClear(pMsgBody->msgInfo.pData); if (pMsgBody->paramFreeFp) { @@ -243,7 +242,8 @@ void destroyQueryExecRes(SExecResult* pRes) { break; } case TDMT_VND_SUBMIT: { - tFreeSSubmitRsp((SSubmitRsp*)pRes->res); + tDestroySSubmitRsp2((SSubmitRsp2*)pRes->res, TSDB_MSG_FLG_DECODE); + taosMemoryFreeClear(pRes->res); break; } case TDMT_SCH_QUERY: @@ -393,7 +393,7 @@ char* parseTagDatatoJson(void* p) { } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { - ASSERT(0); + goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); @@ -412,7 +412,7 @@ char* parseTagDatatoJson(void* p) { } cJSON_AddItemToObject(json, tagJsonKey, value); } else { - ASSERT(0); + goto end; } } string = cJSON_PrintUnformatted(json); @@ -420,7 +420,7 @@ end: cJSON_Delete(json); taosArrayDestroy(pTagVals); if (string == NULL) { - string = strdup(TSDB_DATA_NULL_STR_L); + string = taosStrdup(TSDB_DATA_NULL_STR_L); } return string; } @@ -447,7 +447,6 @@ int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } - void freeVgInfo(SDBVgInfo* vgInfo) { if (NULL == vgInfo) { return; @@ -459,7 +458,6 @@ void freeVgInfo(SDBVgInfo* vgInfo) { taosMemoryFreeClear(vgInfo); } - int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -498,3 +496,53 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + +int32_t cloneSVreateTbReq(SVCreateTbReq* pSrc, SVCreateTbReq** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); + if (NULL == *pDst) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pDst)->flags = pSrc->flags; + if (pSrc->name) { + (*pDst)->name = taosStrdup(pSrc->name); + } + (*pDst)->uid = pSrc->uid; + (*pDst)->ctime = pSrc->ctime; + (*pDst)->ttl = pSrc->ttl; + (*pDst)->commentLen = pSrc->commentLen; + if (pSrc->comment) { + (*pDst)->comment = taosStrdup(pSrc->comment); + } + (*pDst)->type = pSrc->type; + + if (pSrc->type == TSDB_CHILD_TABLE) { + if (pSrc->ctb.stbName) { + (*pDst)->ctb.stbName = taosStrdup(pSrc->ctb.stbName); + } + (*pDst)->ctb.tagNum = pSrc->ctb.tagNum; + (*pDst)->ctb.suid = pSrc->ctb.suid; + if (pSrc->ctb.tagName) { + (*pDst)->ctb.tagName = taosArrayDup(pSrc->ctb.tagName, NULL); + } + STag* pTag = (STag*)pSrc->ctb.pTag; + if (pTag) { + (*pDst)->ctb.pTag = taosMemoryMalloc(pTag->len); + memcpy((*pDst)->ctb.pTag, pTag, pTag->len); + } + } else { + (*pDst)->ntb.schemaRow.nCols = pSrc->ntb.schemaRow.nCols; + (*pDst)->ntb.schemaRow.version = pSrc->ntb.schemaRow.nCols; + if (pSrc->ntb.schemaRow.nCols > 0 && pSrc->ntb.schemaRow.pSchema) { + (*pDst)->ntb.schemaRow.pSchema = taosMemoryMalloc(pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + memcpy((*pDst)->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.pSchema, pSrc->ntb.schemaRow.nCols * sizeof(SSchema)); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 2e4b000761220e4d49a3c03778efe89effebed8a..d2934e1ff86d964cc80e391e53052c25900b0d4a 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -527,7 +527,7 @@ int32_t queryProcessGetSerVerRsp(void *output, char *msg, int32_t msgSize) { return code; } - *(char **)output = strdup(out.ver); + *(char **)output = taosStrdup(out.ver); return code; } diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index 283fc7aa10884d408aa3daeaeeab9ff0007f46df..f198b73c7d880c032064d0ea5edece3908af8351 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -336,34 +336,34 @@ typedef struct SQWorkerMgmt { #define QW_LOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define QW_UNLOCK(type, _lock) \ do { \ if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index e2db0b8319e1db910c3e1d84e9c5663a772b9c6b..ec4049a3eb45c2c274a58837e9f06b1af940e631 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -18,13 +18,12 @@ SQWorkerMgmt gQwMgmt = { .qwNum = 0, }; - int32_t qwStopAllTasks(SQWorker *mgmt) { uint64_t qId, tId, sId; int32_t eId; int64_t rId = 0; - void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); while (pIter) { SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; void *key = taosHashGetKey(pIter, NULL); @@ -49,7 +48,7 @@ int32_t qwStopAllTasks(SQWorker *mgmt) { QW_TASK_DLOG_E("task running, async killed"); } else if (QW_FETCH_RUNNING(ctx)) { QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED); - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); QW_TASK_DLOG_E("task fetching, update drop received"); } else { qwDropTask(QW_FPARAMS()); @@ -192,7 +191,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { size_t numOfResBlock = taosArrayGetSize(pResList); for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); - ASSERT(pRes->info.rows > 0); SInputData inputData = {.pData = pRes}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); @@ -248,7 +246,7 @@ _return: } bool qwTaskNotInExec(SQWTaskCtx *ctx) { - qTaskInfo_t taskHandle = ctx->taskHandle; + qTaskInfo_t taskHandle = ctx->taskHandle; if (NULL == taskHandle || !qTaskIsExecuting(taskHandle)) { return true; } @@ -804,7 +802,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } QW_LOCK(QW_WRITE, &ctx->lock); - if (atomic_load_8((int8_t*)&ctx->queryEnd) || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) { + if (atomic_load_8((int8_t *)&ctx->queryEnd) || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || + code) { // Note: query is not running anymore QW_SET_PHASE(ctx, QW_PHASE_POST_CQUERY); QW_UNLOCK(QW_WRITE, &ctx->lock); @@ -893,6 +892,9 @@ _return: qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), dataLen); + } else { + qwFreeFetchRsp(rsp); + rsp = NULL; } } @@ -1136,7 +1138,6 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } - int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, void **qWorkerMgmt, const SMsgCb *pMsgCb) { if (NULL == qWorkerMgmt || (pMsgCb && pMsgCb->mgmt == NULL)) { qError("invalid param to init qworker"); @@ -1235,7 +1236,7 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) { SQWorker *mgmt = (SQWorker *)qWorkerMgmt; QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash)); - + atomic_store_8(&mgmt->nodeStopped, 1); (void)qwStopAllTasks(mgmt); diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index d91b2ebd6bc059da42f85ff78a5f677168de3edb..144a9536b94184598fa7e2d6bd25137acb5c69fd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -15,7 +15,7 @@ #include #include "os.h" #include "thash.h" -//#include "queryLog.h" +// #include "queryLog.h" #include "filter.h" #include "filterInt.h" #include "functionMgt.h" @@ -123,36 +123,16 @@ int8_t filterGetRangeCompFuncFromOptrs(uint8_t optr, uint8_t optr2) { return -1; } -__compar_fn_t gDataCompare[] = {compareInt32Val, - compareInt8Val, - compareInt16Val, - compareInt64Val, - compareFloatVal, - compareDoubleVal, - compareLenPrefixedStr, - comparestrPatternMatch, - compareChkInString, - comparewcsPatternMatch, - compareLenPrefixedWStr, - compareUint8Val, - compareUint16Val, - compareUint32Val, - compareUint64Val, - setChkInBytes1, - setChkInBytes2, - setChkInBytes4, - setChkInBytes8, - comparestrRegexMatch, - comparestrRegexNMatch, - setChkNotInBytes1, - setChkNotInBytes2, - setChkNotInBytes4, - setChkNotInBytes8, - compareChkNotInString, - comparestrPatternNMatch, - comparewcsPatternNMatch, - comparewcsRegexMatch, - comparewcsRegexNMatch,}; +__compar_fn_t gDataCompare[] = { + compareInt32Val, compareInt8Val, compareInt16Val, compareInt64Val, + compareFloatVal, compareDoubleVal, compareLenPrefixedStr, comparestrPatternMatch, + compareChkInString, comparewcsPatternMatch, compareLenPrefixedWStr, compareUint8Val, + compareUint16Val, compareUint32Val, compareUint64Val, setChkInBytes1, + setChkInBytes2, setChkInBytes4, setChkInBytes8, comparestrRegexMatch, + comparestrRegexNMatch, setChkNotInBytes1, setChkNotInBytes2, setChkNotInBytes4, + setChkNotInBytes8, compareChkNotInString, comparestrPatternNMatch, comparewcsPatternNMatch, + comparewcsRegexMatch, comparewcsRegexNMatch, +}; __compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32, compareInt8Int64, compareInt8Float, compareInt8Double}; @@ -341,7 +321,7 @@ __compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr) { if (TSDB_DATA_TYPE_NULL == rType || TSDB_DATA_TYPE_JSON == rType) { return NULL; } - + switch (lType) { case TSDB_DATA_TYPE_TINYINT: { if (IS_SIGNED_NUMERIC_TYPE(rType) || IS_FLOAT_TYPE(rType)) { @@ -519,7 +499,7 @@ int32_t filterReuseRangeCtx(SFilterRangeCtx *ctx, int32_t type, int32_t options) int32_t filterConvertRange(SFilterRangeCtx *cur, SFilterRange *ra, bool *notNull) { int64_t tmp = 0; - + if (!FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { int32_t sr = cur->pCompareFunc(&ra->s, getDataMin(cur->type, &tmp)); if (sr == 0) { @@ -704,7 +684,7 @@ int32_t filterAddRangeImpl(void *h, SFilterRange *ra, int32_t optr) { int32_t filterAddRange(void *h, SFilterRange *ra, int32_t optr) { SFilterRangeCtx *ctx = (SFilterRangeCtx *)h; - int64_t tmp = 0; + int64_t tmp = 0; if (FILTER_GET_FLAG(ra->sflag, RANGE_FLG_NULL)) { SIMPLE_COPY_VALUES(&ra->s, getDataMin(ctx->type, &tmp)); @@ -991,7 +971,7 @@ int32_t filterAddField(SFilterInfo *info, void *desc, void **data, int32_t type, bool freeIfExists, int16_t *srcFlag) { int32_t idx = -1; uint32_t *num; - bool sameBuf = false; + bool sameBuf = false; num = &info->fields[type].num; @@ -1251,13 +1231,13 @@ int32_t filterAddUnitFromUnit(SFilterInfo *dst, SFilterInfo *src, SFilterUnit *u SFilterField *t = FILTER_UNIT_LEFT_FIELD(src, u); if (u->right.type == FLD_TYPE_VALUE) { - void *data = FILTER_UNIT_VAL_DATA(src, u); + void *data = FILTER_UNIT_VAL_DATA(src, u); SFilterField *rField = FILTER_UNIT_RIGHT_FIELD(src, u); if (IS_VAR_DATA_TYPE(type)) { if (FILTER_UNIT_OPTR(u) == OP_TYPE_IN) { - filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, - false, &rField->flag); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. + filterAddField(dst, NULL, &data, FLD_TYPE_VALUE, &right, POINTER_BYTES, false, + &rField->flag); // POINTER_BYTES should be sizeof(SHashObj), but POINTER_BYTES is also right. t = FILTER_GET_FIELD(dst, right); FILTER_SET_FLAG(t->flag, FLD_DATA_IS_HASH); @@ -3208,7 +3188,6 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, SColumnInfoData *pRes for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - if (colDataIsNull_s((SColumnInfoData *)info->cunits[uidx].colData, i)) { p[i] = 0; all = false; @@ -3770,31 +3749,31 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { return DEAL_RES_CONTINUE; } -/* - if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { - return DEAL_RES_CONTINUE; - } + /* + if (!FILTER_GET_FLAG(stat->info->options, FLT_OPTION_TIMESTAMP)) { + return DEAL_RES_CONTINUE; + } - if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != valueNode->node.resType.type) { - return DEAL_RES_CONTINUE; - } + if (TSDB_DATA_TYPE_BINARY != valueNode->node.resType.type && TSDB_DATA_TYPE_NCHAR != + valueNode->node.resType.type) { return DEAL_RES_CONTINUE; + } - if (stat->precision < 0) { - int32_t code = fltAddValueNodeToConverList(stat, valueNode); - if (code) { - stat->code = code; - return DEAL_RES_ERROR; - } + if (stat->precision < 0) { + int32_t code = fltAddValueNodeToConverList(stat, valueNode); + if (code) { + stat->code = code; + return DEAL_RES_ERROR; + } - return DEAL_RES_CONTINUE; - } + return DEAL_RES_CONTINUE; + } - int32_t code = sclConvertToTsValueNode(stat->precision, valueNode); - if (code) { - stat->code = code; - return DEAL_RES_ERROR; - } -*/ + int32_t code = sclConvertToTsValueNode(stat->precision, valueNode); + if (code) { + stat->code = code; + return DEAL_RES_ERROR; + } + */ return DEAL_RES_CONTINUE; } @@ -3940,7 +3919,7 @@ EDealRes fltReviseRewriter(SNode **pNode, void *pContext) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->node.resType.type); + int32_t type = vectorGetConvertType(refNode->node.resType.type, listNode->node.resType.type); if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; @@ -3964,14 +3943,14 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode **pNode, SFltTreeStat *pStat) { FLT_ERR_JRET(pStat->code); -/* - int32_t nodeNum = taosArrayGetSize(pStat->nodeList); - for (int32_t i = 0; i < nodeNum; ++i) { - SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); + /* + int32_t nodeNum = taosArrayGetSize(pStat->nodeList); + for (int32_t i = 0; i < nodeNum; ++i) { + SValueNode *valueNode = *(SValueNode **)taosArrayGet(pStat->nodeList, i); - FLT_ERR_JRET(sclConvertToTsValueNode(pStat->precision, valueNode)); - } -*/ + FLT_ERR_JRET(sclConvertToTsValueNode(pStat->precision, valueNode)); + } + */ _return: diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 14eb21565bcdaa7131c58d1fb8a09728802df616..85b952937fbc356879adc1b53a059767b9164e5f 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -479,34 +479,34 @@ extern SSchedulerMgmt schMgmt; #define SCH_LOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before read lock"); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) > 0); \ + ASSERTS(atomic_load_32(_lock) > 0, "invalid lock value after read lock"); \ } else { \ - assert(atomic_load_32(_lock) >= 0); \ + ASSERTS(atomic_load_32(_lock) >= 0, "invalid lock value before write lock"); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32(_lock) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ } \ } while (0) #define SCH_UNLOCK(type, _lock) \ do { \ if (SCH_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ } else { \ - assert(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY); \ + ASSERTS(atomic_load_32((_lock)) & TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ SCH_LOCK_DEBUG("SCH WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ } \ } while (0) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 6a8f81f8c78a0146396371d79f4c8ed95e73b1fb..980a8ac6a193e32c38e64f0b323c5311ddacef55 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -234,7 +234,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } SSchTask *pTask = taosArrayGet(pLevel->subTasks, 0); - if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType) { + if (SUBPLAN_TYPE_MODIFY != pTask->plan->subplanType || EXPLAIN_MODE_DISABLE != pJob->attr.explainMode) { pJob->attr.needFetch = true; } } @@ -484,7 +484,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { if (TSDB_CODE_SCH_IGNORE_ERROR == errCode) { return TSDB_CODE_SCH_IGNORE_ERROR; } - + schUpdateJobErrCode(pJob, errCode); int32_t code = atomic_load_32(&pJob->errCode); @@ -537,7 +537,9 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); - schProcessOnDataFetched(pJob); + if (!SCH_IS_INSERT_JOB(pJob)) { + schProcessOnDataFetched(pJob); + } return TSDB_CODE_SUCCESS; } @@ -682,7 +684,7 @@ void schFreeJobImpl(void *job) { int32_t schJobFetchRows(SSchJob *pJob) { int32_t code = 0; - if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC)) { + if (!(pJob->attr.explainMode == EXPLAIN_MODE_STATIC) && !(SCH_IS_EXPLAIN_JOB(pJob) && SCH_IS_INSERT_JOB(pJob))) { SCH_ERR_RET(schLaunchFetchTask(pJob)); if (schChkCurrentOp(pJob, SCH_OP_FETCH, true)) { @@ -714,7 +716,7 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { pJob->attr.localExec = pReq->localReq; pJob->conn = *pReq->pConn; if (pReq->sql) { - pJob->sql = strdup(pReq->sql); + pJob->sql = taosStrdup(pReq->sql); } pJob->pDag = pReq->pDag; pJob->allocatorRefId = nodesMakeAllocatorWeakRef(pReq->allocatorRefId); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 9c4ed65dd205f859b71ed3b0a56001bd5b542e4a..6f4130fd9ffe2814f0dd30655a58aefc92801a13 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -261,46 +261,51 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa if (msg) { SDecoder coder = {0}; - SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp)); + SSubmitRsp2 *rsp = taosMemoryMalloc(sizeof(*rsp)); tDecoderInit(&coder, msg, msgSize); - code = tDecodeSSubmitRsp(&coder, rsp); + code = tDecodeSSubmitRsp2(&coder, rsp); + tDecoderClear(&coder); if (code) { - SCH_TASK_ELOG("decode submitRsp failed, code:%d", code); - tFreeSSubmitRsp(rsp); + SCH_TASK_ELOG("tDecodeSSubmitRsp2 failed, code:%d", code); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); + taosMemoryFree(rsp); SCH_ERR_JRET(code); } - if (rsp->nBlocks > 0) { - for (int32_t i = 0; i < rsp->nBlocks; ++i) { - SSubmitBlkRsp *blk = rsp->pBlocks + i; - if (TSDB_CODE_SUCCESS != blk->code) { - code = blk->code; - tFreeSSubmitRsp(rsp); - SCH_ERR_JRET(code); + atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); + + int32_t createTbRspNum = taosArrayGetSize(rsp->aCreateTbRsp); + SCH_TASK_DLOG("submit succeed, affectedRows:%d, createTbRspNum:%d", rsp->affectedRows, createTbRspNum); + + if (rsp->aCreateTbRsp && taosArrayGetSize(rsp->aCreateTbRsp) > 0) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->execRes.res) { + SSubmitRsp2 *sum = pJob->execRes.res; + sum->affectedRows += rsp->affectedRows; + if (sum->aCreateTbRsp) { + taosArrayAddAll(sum->aCreateTbRsp, rsp->aCreateTbRsp); + taosArrayDestroy(rsp->aCreateTbRsp); + } else { + TSWAP(sum->aCreateTbRsp, rsp->aCreateTbRsp); } + taosMemoryFree(rsp); + } else { + pJob->execRes.res = rsp; + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - } - - atomic_add_fetch_64(&pJob->resNumOfRows, rsp->affectedRows); - SCH_TASK_DLOG("submit succeed, affectedRows:%d, blocks:%d", rsp->affectedRows, rsp->nBlocks); - - SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->execRes.res) { - SSubmitRsp *sum = pJob->execRes.res; - sum->affectedRows += rsp->affectedRows; - sum->nBlocks += rsp->nBlocks; - if (rsp->nBlocks > 0 && rsp->pBlocks) { - sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); - memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks)); + pJob->execRes.numOfBytes += pTask->msgLen; + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + } else { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + pJob->execRes.numOfBytes += pTask->msgLen; + if (NULL == pJob->execRes.res) { + TSWAP(pJob->execRes.res, rsp); + pJob->execRes.msgType = TDMT_VND_SUBMIT; } - taosMemoryFree(rsp->pBlocks); + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + tDestroySSubmitRsp2(rsp, TSDB_MSG_FLG_DECODE); taosMemoryFree(rsp); - } else { - pJob->execRes.res = rsp; - pJob->execRes.msgType = TDMT_VND_SUBMIT; } - pJob->execRes.numOfBytes += pTask->msgLen; - SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } taosMemoryFreeClear(msg); @@ -336,6 +341,14 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + if (taosArrayGetSize(pTask->parents) == 0 && SCH_IS_EXPLAIN_JOB(pJob) && SCH_IS_INSERT_JOB(pJob)) { + SRetrieveTableRsp *pRsp = NULL; + SCH_ERR_JRET(qExecExplainEnd(pJob->explainCtx, &pRsp)); + if (pRsp) { + SCH_ERR_JRET(schProcessOnExplainDone(pJob, pTask, pRsp)); + } + } + SQueryTableRsp rsp = {0}; if (tDeserializeSQueryTableRsp(msg, msgSize, &rsp) < 0) { SCH_TASK_ELOG("tDeserializeSQueryTableRsp failed, msgSize:%d", msgSize); @@ -874,7 +887,7 @@ int32_t schUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, SQueryNodeAddr *addr } else { pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = addr->nodeId; - pMsgSendInfo->target.dbFName = strdup(pTask->plan->dbFName); + pMsgSendInfo->target.dbFName = taosStrdup(pTask->plan->dbFName); } return TSDB_CODE_SUCCESS; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index efd19074da1b2f51e1217b3f7ab359f2b4a33c95..670cfbead1e180061fe0f972290351125eb9852c 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include "streamInc.h" int32_t tEncodeSStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { @@ -192,3 +193,4 @@ int32_t streamProcessCheckpointRsp(SStreamMeta* pMeta, SStreamTask* pTask, SStre // set status normal return 0; } +#endif diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 12ce483c0c4d8663f63a72647f704bf0d501e818..8baebaee4224e6b9af5d2eacf62910412171bc63 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -26,7 +26,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock ASSERT(pReq->blockNum == taosArrayGetSize(pReq->dataLen)); for (int32_t i = 0; i < blockNum; i++) { - SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); blockDecode(pDataBlock, pRetrieve->data); // TODO: refactor @@ -65,12 +65,13 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { - SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { + SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); if (pDataSubmit->dataRef == NULL) goto FAIL; - pDataSubmit->data = pReq; + pDataSubmit->submit = submit; *pDataSubmit->dataRef = 1; pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; return pDataSubmit; @@ -79,47 +80,49 @@ FAIL: return NULL; } -SStreamMergedSubmit* streamMergedSubmitNew() { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)taosAllocateQitem(sizeof(SStreamMergedSubmit), DEF_QITEM, 0); +SStreamMergedSubmit2* streamMergedSubmitNew() { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) return NULL; - pMerged->reqs = taosArrayInit(0, sizeof(void*)); + pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->reqs == NULL) goto FAIL; + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; FAIL: - if (pMerged->reqs) taosArrayDestroy(pMerged->reqs); + if (pMerged->submits) taosArrayDestroy(pMerged->submits); if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); taosFreeQitem(pMerged); return NULL; } -int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) { +int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { taosArrayPush(pMerged->dataRefs, &pSubmit->dataRef); - taosArrayPush(pMerged->reqs, &pSubmit->data); + taosArrayPush(pMerged->submits, &pSubmit->submit); pMerged->ver = pSubmit->ver; return 0; } -static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit* pDataSubmit) { +static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) { atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit) { - SStreamDataSubmit* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM, 0); +SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { + SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pSubmitClone == NULL) { return NULL; } streamDataSubmitRefInc(pSubmit); - memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit)); + memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit) { +void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); ASSERT(ref >= 0); if (ref == 0) { - taosMemoryFree(pDataSubmit->data); + taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->dataRef); } } @@ -134,16 +137,16 @@ SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)dst; - SStreamDataSubmit* pBlockSrc = (SStreamDataSubmit*)elem; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; streamMergeSubmit(pMerged, pBlockSrc); taosFreeQitem(elem); return dst; } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamMergedSubmit* pMerged = streamMergedSubmitNew(); + SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); taosFreeQitem(dst); taosFreeQitem(elem); return (SStreamQueueItem*)pMerged; @@ -161,22 +164,22 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + streamDataSubmitRefDec((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerge = (SStreamMergedSubmit*)data; - int32_t sz = taosArrayGetSize(pMerge->reqs); + SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; + int32_t sz = taosArrayGetSize(pMerge->submits); for (int32_t i = 0; i < sz; i++) { int32_t* pRef = taosArrayGetP(pMerge->dataRefs, i); int32_t ref = atomic_sub_fetch_32(pRef, 1); ASSERT(ref >= 0); if (ref == 0) { - void* dataStr = taosArrayGetP(pMerge->reqs, i); - taosMemoryFree(dataStr); + SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i); + taosMemoryFree(pSubmit->msgStr); taosMemoryFree(pRef); } } - taosArrayDestroy(pMerge->reqs); + taosArrayDestroy(pMerge->submits); taosArrayDestroy(pMerge->dataRefs); taosFreeQitem(pMerge); } else if (type == STREAM_INPUT__REF_DATA_BLOCK) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 7bf19cec63f89c7c17e79cc6af3a76566cbcaf51..9226d6ebb8dabc349c333470b0fc2d44396c2c0b 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -28,17 +28,18 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - const SStreamDataSubmit* pSubmit = (const SStreamDataSubmit*)data; - qDebug("task %d %p set submit input %p %p %d 1", pTask->taskId, pTask, pSubmit, pSubmit->data, *pSubmit->dataRef); - qSetMultiStreamInput(exec, pSubmit->data, 1, STREAM_INPUT__DATA_SUBMIT); + const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; + qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + pSubmit->submit.msgLen, pSubmit->submit.ver); + qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qDebug("task %d %p set ssdata input", pTask->taskId, pTask); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { - const SStreamMergedSubmit* pMerged = (const SStreamMergedSubmit*)data; - SArray* blocks = pMerged->reqs; + const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; + SArray* blocks = pMerged->submits; qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { @@ -113,12 +114,14 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + taosArrayDestroy(pRes); return 0; } SSDataBlock* output = NULL; uint64_t ts = 0; if (qExecTask(exec, &output, &ts) < 0) { + taosArrayDestroy(pRes); return -1; } if (output == NULL) { @@ -135,7 +138,11 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - if (++batchCnt >= batchSz) break; + batchCnt++; + + qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz); + + if (batchCnt >= batchSz) break; } if (taosArrayGetSize(pRes) == 0) { taosArrayDestroy(pRes); @@ -153,6 +160,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { streamTaskOutput(pTask, qRes); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt); streamDispatch(pTask); } if (finished) break; @@ -262,11 +270,11 @@ int32_t streamExecForAll(SStreamTask* pTask) { qRes->blocks = pRes; if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)input; + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit* pMerged = (SStreamMergedSubmit*)input; + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 63527e2b1c55af8814eed0551088fa1026ff55f2..66d98e90bfaa7f81e5c985de205d245c9a3b0120 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -26,7 +26,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF int32_t len = strlen(path) + 20; char* streamPath = taosMemoryCalloc(1, len); sprintf(streamPath, "%s/%s", path, "stream"); - pMeta->path = strdup(streamPath); + pMeta->path = taosStrdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { taosMemoryFree(streamPath); goto _err; @@ -171,6 +171,7 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { } #endif +#if 0 SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { @@ -180,6 +181,7 @@ SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { return NULL; } } +#endif SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 7eafcdc93ea20b96140391c0e555ff5caf5bff93..882fba718bb3bc6cde73cea38c151e849936a155 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -46,6 +46,7 @@ void streamQueueClose(SStreamQueue* queue) { taosMemoryFree(queue); } +#if 0 bool streamQueueResEmpty(const SStreamQueueRes* pRes) { // return true; @@ -101,3 +102,4 @@ SStreamQueueRes streamQueueGetRes(SStreamQueue1* pQueue) { if (pNode) return streamQueueBuildRes(pNode); return (SStreamQueueRes){0}; } +#endif diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index b4e44cc5b7009d91e93011ee9f8cdcd79238d1c1..a2b3e20dbfbab336ef50ef786daff4ccba95858a 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -179,6 +179,11 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int goto _err; } + if (tdbTbOpen("partag.state.db", sizeof(int64_t), -1, NULL, pState->pTdbState->db, &pState->pTdbState->pParTagDb, 0) < + 0) { + goto _err; + } + if (streamStateBegin(pState) < 0) { goto _err; } @@ -193,6 +198,7 @@ _err: tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pParNameDb); + tdbTbClose(pState->pTdbState->pParTagDb); tdbClose(pState->pTdbState->db); streamStateDestroy(pState); return NULL; @@ -206,6 +212,7 @@ void streamStateClose(SStreamState* pState) { tdbTbClose(pState->pTdbState->pFillStateDb); tdbTbClose(pState->pTdbState->pSessionStateDb); tdbTbClose(pState->pTdbState->pParNameDb); + tdbTbClose(pState->pTdbState->pParTagDb); tdbClose(pState->pTdbState->db); streamStateDestroy(pState); @@ -840,10 +847,17 @@ _end: return res; } +int32_t streamStatePutParTag(SStreamState* pState, int64_t groupId, const void* tag, int32_t tagLen) { + return tdbTbUpsert(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tag, tagLen, pState->pTdbState->txn); +} + +int32_t streamStateGetParTag(SStreamState* pState, int64_t groupId, void** tagVal, int32_t* tagLen) { + return tdbTbGet(pState->pTdbState->pParTagDb, &groupId, sizeof(int64_t), tagVal, tagLen); +} + int32_t streamStatePutParName(SStreamState* pState, int64_t groupId, const char tbname[TSDB_TABLE_NAME_LEN]) { - tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, - pState->pTdbState->txn); - return 0; + return tdbTbUpsert(pState->pTdbState->pParNameDb, &groupId, sizeof(int64_t), tbname, TSDB_TABLE_NAME_LEN, + pState->pTdbState->txn); } int32_t streamStateGetParName(SStreamState* pState, int64_t groupId, void** pVal) { diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index 1ce4a35dff6441a9ec289d1cb94eb24743122762..be12c72d004711e6c126792b5a7cfe4053ad1f18 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -85,9 +85,7 @@ static int64_t adjustWatermark(int64_t adjInterval, int64_t originInt, int64_t w watermark = TMAX(originInt / adjInterval, 1) * adjInterval; } else if (watermark > MAX_NUM_SCALABLE_BF * adjInterval) { watermark = MAX_NUM_SCALABLE_BF * adjInterval; - }/* else if (watermark < MIN_NUM_SCALABLE_BF * adjInterval) { - watermark = MIN_NUM_SCALABLE_BF * adjInterval; - }*/ // Todo(liuyao) save window info to tdb + } return watermark; } diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index a823cfda0b76cda1a60650a8208c5cfa4a39a160..a1de2ee71ac12fba4a43676c589c9acb323492ac 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -68,7 +68,7 @@ void syncNodeLogReplMgrDestroy(SSyncNode* pNode); // access static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { - return (1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; + return ((int64_t)1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; } static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index 58944eb08c88800e90294bc625733d7f3c7e2051..a39e043c52a0016a6e7242572667a9d02d62be5e 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -49,39 +49,6 @@ static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) { return pEntry->originalRpcType == TDMT_SYNC_NOOP; } -typedef struct SRaftEntryHashCache { - SHashObj* pEntryHash; - int32_t maxCount; - int32_t currentCount; - TdThreadMutex mutex; - SSyncNode* pSyncNode; -} SRaftEntryHashCache; - -SRaftEntryHashCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); -void raftCacheDestroy(SRaftEntryHashCache* pCache); -int32_t raftCachePutEntry(struct SRaftEntryHashCache* pCache, SSyncRaftEntry* pEntry); -int32_t raftCacheGetEntry(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); -int32_t raftCacheGetEntryP(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); -int32_t raftCacheDelEntry(struct SRaftEntryHashCache* pCache, SyncIndex index); -int32_t raftCacheGetAndDel(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); -int32_t raftCacheClear(struct SRaftEntryHashCache* pCache); - -typedef struct SRaftEntryCache { - SSkipList* pSkipList; - int32_t maxCount; - int32_t currentCount; - int32_t refMgr; - TdThreadMutex mutex; - SSyncNode* pSyncNode; -} SRaftEntryCache; - -SRaftEntryCache* raftEntryCacheCreate(SSyncNode* pSyncNode, int32_t maxCount); -void raftEntryCacheDestroy(SRaftEntryCache* pCache); -int32_t raftEntryCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry); -int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); -int32_t raftEntryCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry); -int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count); - #ifdef __cplusplus } #endif diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index b04bcb86c69de5088e53f2852d24430a722979a0..bd1dae54d956a9a722d70eac30c596989d117cf3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -104,6 +104,8 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { SyncAppendEntries* pMsg = pRpcMsg->pCont; SRpcMsg rpcRsp = {0}; bool accepted = false; + SSyncRaftEntry* pEntry = NULL; + // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) { syncLogRecvAppendEntries(ths, pMsg, "not in my config"); @@ -137,14 +139,13 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncNodeStepDown(ths, pMsg->term); syncNodeResetElectTimer(ths); - if (pMsg->dataLen < (int32_t)sizeof(SSyncRaftEntry)) { + if (pMsg->dataLen < sizeof(SSyncRaftEntry)) { sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d", ths->vgId, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); goto _IGNORE; } - SSyncRaftEntry* pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); - + pEntry = syncBuildRaftEntryFromAppendEntries(pMsg); if (pEntry == NULL) { sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr()); goto _IGNORE; @@ -191,5 +192,6 @@ _out: _IGNORE: rpcFreeCont(rpcRsp.pCont); + syncEntryDestroy(pEntry); return 0; } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index f81699b9f6de4f0d7881cee7d42c6dfefd7acd71..a60f43cd5ed053fc00eba4b97c32e908698ea499 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -40,7 +40,7 @@ // int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { - SyncAppendEntriesReply* pMsg = pRpcMsg->pCont; + SyncAppendEntriesReply* pMsg = (SyncAppendEntriesReply*)pRpcMsg->pCont; int32_t ret = 0; // if already drop replica, do not process diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fc0d235df97ef13db4a57fae86855e2441b522b9..b07c05dcfe5ddb20d0b59bf17db37cd6a4c1a359 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -731,14 +731,25 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; + bool updated = false; sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); for (int32_t i = 0; i < pCfg->replicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; - tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { + updated = true; + } sInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); } + if (updated) { + sInfo("vgId:%d, save config info since dnode info changed", pSyncNode->vgId); + if (syncWriteCfgFile(pSyncNode) != 0) { + sError("vgId:%d, failed to write sync cfg file on dnode info updated", pSyncNode->vgId); + goto _error; + } + } + pSyncNode->pWal = pSyncInfo->pWal; pSyncNode->msgcb = pSyncInfo->msgcb; pSyncNode->syncSendMSg = pSyncInfo->syncSendMSg; @@ -1067,29 +1078,18 @@ int32_t syncNodeStartStandBy(SSyncNode* pSyncNode) { } void syncNodePreClose(SSyncNode* pSyncNode) { - if (pSyncNode != NULL && pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpApplyQueueItems != NULL) { - while (1) { - int32_t aqItems = pSyncNode->pFsm->FpApplyQueueItems(pSyncNode->pFsm); - sTrace("vgId:%d, pre close, %d items in apply queue", pSyncNode->vgId, aqItems); - if (aqItems == 0 || aqItems == -1) { - break; - } - taosMsleep(20); - } - } - -#if 0 - if (pSyncNode->pNewNodeReceiver != NULL) { - if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { - snapshotReceiverStop(pSyncNode->pNewNodeReceiver); + ASSERT(pSyncNode != NULL); + ASSERT(pSyncNode->pFsm != NULL); + ASSERT(pSyncNode->pFsm->FpApplyQueueItems != NULL); + + while (1) { + int32_t aqItems = pSyncNode->pFsm->FpApplyQueueItems(pSyncNode->pFsm); + sTrace("vgId:%d, pre close, %d items in apply queue", pSyncNode->vgId, aqItems); + if (aqItems == 0 || aqItems == -1) { + break; } - - sDebug("vgId:%d, snapshot receiver destroy while preclose sync node, data:%p", pSyncNode->vgId, - pSyncNode->pNewNodeReceiver); - snapshotReceiverDestroy(pSyncNode->pNewNodeReceiver); - pSyncNode->pNewNodeReceiver = NULL; + taosMsleep(20); } -#endif // stop elect timer syncNodeStopElectTimer(pSyncNode); @@ -1402,7 +1402,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - sNInfo(pSyncNode, "begin do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum, + sNInfo(pSyncNode, "begin do config change, from %d to %d, replicas:%d", pSyncNode->vgId, oldConfig.replicaNum, pNewConfig->replicaNum); if (IamInNew) { @@ -1683,8 +1683,7 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { #endif // close receiver - if (pSyncNode != NULL && pSyncNode->pNewNodeReceiver != NULL && - snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { + if (snapshotReceiverIsStart(pSyncNode->pNewNodeReceiver)) { snapshotReceiverStop(pSyncNode->pNewNodeReceiver); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index e2b039a2e489e1dc9d3c6887c11c188523391640..faa44a626c96f0835b0df2885166e15851f0b632 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -910,7 +910,7 @@ int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNo int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < (SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + if (timeDiffMs > 0 && timeDiffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { pMgr->retryBackoff -= 1; } } @@ -937,10 +937,6 @@ SSyncLogReplMgr* syncLogReplMgrCreate() { ASSERT(pMgr->size == TSDB_SYNC_LOG_BUFFER_SIZE); return pMgr; - -_err: - taosMemoryFree(pMgr); - return NULL; } void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 806949c81e2ad8598a7d2901769c53a0bfa2daa8..f780e255ce77eb485ffcd1de8d078718dc1973a5 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -224,7 +224,7 @@ _OVER: int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex) { SRaftCfg *pCfg = &pNode->raftCfg; - if (pCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT) { + if (pCfg->configIndexCount < MAX_CONFIG_INDEX_COUNT) { return -1; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 623f1b77a428806a859c4ba5ffc42404ad9d0c84..3e63e2fb8ea8ae3de5ddded5a7307d4037e25f7d 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -102,344 +102,3 @@ void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } - -SRaftEntryHashCache* raftCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { - SRaftEntryHashCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryHashCache)); - if (pCache == NULL) { - sError("vgId:%d, raft cache create error", pSyncNode->vgId); - return NULL; - } - - pCache->pEntryHash = - taosHashInit(sizeof(SyncIndex), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - if (pCache->pEntryHash == NULL) { - sError("vgId:%d, raft cache create hash error", pSyncNode->vgId); - return NULL; - } - - taosThreadMutexInit(&(pCache->mutex), NULL); - pCache->maxCount = maxCount; - pCache->currentCount = 0; - pCache->pSyncNode = pSyncNode; - - return pCache; -} - -void raftCacheDestroy(SRaftEntryHashCache* pCache) { - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - taosHashCleanup(pCache->pEntryHash); - taosThreadMutexUnlock(&pCache->mutex); - taosThreadMutexDestroy(&(pCache->mutex)); - taosMemoryFree(pCache); - } -} - -// success, return 1 -// max count, return 0 -// error, return -1 -int32_t raftCachePutEntry(struct SRaftEntryHashCache* pCache, SSyncRaftEntry* pEntry) { - taosThreadMutexLock(&pCache->mutex); - - if (pCache->currentCount >= pCache->maxCount) { - taosThreadMutexUnlock(&pCache->mutex); - return 0; - } - - taosHashPut(pCache->pEntryHash, &(pEntry->index), sizeof(pEntry->index), pEntry, pEntry->bytes); - ++(pCache->currentCount); - - sNTrace(pCache->pSyncNode, "raft cache add, type:%s,%d, type2:%s,%d, index:%" PRId64 ", bytes:%d", - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, - pEntry->index, pEntry->bytes); - taosThreadMutexUnlock(&pCache->mutex); - return 1; -} - -// success, return 0 -// error, return -1 -// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST -int32_t raftCacheGetEntry(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { - if (ppEntry == NULL) { - return -1; - } - *ppEntry = NULL; - - taosThreadMutexLock(&pCache->mutex); - void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); - if (pTmp != NULL) { - SSyncRaftEntry* pEntry = pTmp; - *ppEntry = taosMemoryMalloc(pEntry->bytes); - memcpy(*ppEntry, pTmp, pEntry->bytes); - - sNTrace(pCache->pSyncNode, "raft cache get, type:%s,%d, type2:%s,%d, index:%" PRId64, - TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), - (*ppEntry)->originalRpcType, (*ppEntry)->index); - taosThreadMutexUnlock(&pCache->mutex); - return 0; - } - - taosThreadMutexUnlock(&pCache->mutex); - terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; - return -1; -} - -// success, return 0 -// error, return -1 -// not exist, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST -int32_t raftCacheGetEntryP(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { - if (ppEntry == NULL) { - return -1; - } - *ppEntry = NULL; - - taosThreadMutexLock(&pCache->mutex); - void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); - if (pTmp != NULL) { - SSyncRaftEntry* pEntry = pTmp; - *ppEntry = pEntry; - - sNTrace(pCache->pSyncNode, "raft cache get, type:%s,%d, type2:%s,%d, index:%" PRId64, - TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), - (*ppEntry)->originalRpcType, (*ppEntry)->index); - taosThreadMutexUnlock(&pCache->mutex); - return 0; - } - - taosThreadMutexUnlock(&pCache->mutex); - terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; - return -1; -} - -int32_t raftCacheDelEntry(struct SRaftEntryHashCache* pCache, SyncIndex index) { - taosThreadMutexLock(&pCache->mutex); - taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); - --(pCache->currentCount); - taosThreadMutexUnlock(&pCache->mutex); - return 0; -} - -int32_t raftCacheGetAndDel(struct SRaftEntryHashCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { - if (ppEntry == NULL) { - return -1; - } - *ppEntry = NULL; - - taosThreadMutexLock(&pCache->mutex); - void* pTmp = taosHashGet(pCache->pEntryHash, &index, sizeof(index)); - if (pTmp != NULL) { - SSyncRaftEntry* pEntry = pTmp; - *ppEntry = taosMemoryMalloc(pEntry->bytes); - memcpy(*ppEntry, pTmp, pEntry->bytes); - - sNTrace(pCache->pSyncNode, "raft cache get-and-del, type:%s,%d, type2:%s,%d, index:%" PRId64, - TMSG_INFO((*ppEntry)->msgType), (*ppEntry)->msgType, TMSG_INFO((*ppEntry)->originalRpcType), - (*ppEntry)->originalRpcType, (*ppEntry)->index); - - taosHashRemove(pCache->pEntryHash, &index, sizeof(index)); - --(pCache->currentCount); - - taosThreadMutexUnlock(&pCache->mutex); - return 0; - } - - taosThreadMutexUnlock(&pCache->mutex); - terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; - return -1; -} - -int32_t raftCacheClear(struct SRaftEntryHashCache* pCache) { - taosThreadMutexLock(&pCache->mutex); - taosHashClear(pCache->pEntryHash); - pCache->currentCount = 0; - taosThreadMutexUnlock(&pCache->mutex); - return 0; -} - -static char* keyFn(const void* pData) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)pData; - return (char*)(&(pEntry->index)); -} - -static int cmpFn(const void* p1, const void* p2) { return memcmp(p1, p2, sizeof(SyncIndex)); } - -static void freeRaftEntry(void* param) { - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)param; - syncEntryDestroy(pEntry); -} - -SRaftEntryCache* raftEntryCacheCreate(SSyncNode* pSyncNode, int32_t maxCount) { - SRaftEntryCache* pCache = taosMemoryMalloc(sizeof(SRaftEntryCache)); - if (pCache == NULL) { - sError("vgId:%d, raft cache create error", pSyncNode->vgId); - return NULL; - } - - pCache->pSkipList = - tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, sizeof(SyncIndex), cmpFn, SL_ALLOW_DUP_KEY, keyFn); - if (pCache->pSkipList == NULL) { - sError("vgId:%d, raft cache create hash error", pSyncNode->vgId); - return NULL; - } - - taosThreadMutexInit(&(pCache->mutex), NULL); - pCache->refMgr = taosOpenRef(10, freeRaftEntry); - pCache->maxCount = maxCount; - pCache->currentCount = 0; - pCache->pSyncNode = pSyncNode; - - return pCache; -} - -void raftEntryCacheDestroy(SRaftEntryCache* pCache) { - if (pCache != NULL) { - taosThreadMutexLock(&pCache->mutex); - tSkipListDestroy(pCache->pSkipList); - if (pCache->refMgr != -1) { - taosCloseRef(pCache->refMgr); - pCache->refMgr = -1; - } - taosThreadMutexUnlock(&pCache->mutex); - taosThreadMutexDestroy(&(pCache->mutex)); - taosMemoryFree(pCache); - } -} - -// success, return 1 -// max count, return 0 -// error, return -1 -int32_t raftEntryCachePutEntry(struct SRaftEntryCache* pCache, SSyncRaftEntry* pEntry) { - taosThreadMutexLock(&pCache->mutex); - - if (pCache->currentCount >= pCache->maxCount) { - taosThreadMutexUnlock(&pCache->mutex); - return 0; - } - - SSkipListNode* pSkipListNode = tSkipListPut(pCache->pSkipList, pEntry); - ASSERT(pSkipListNode != NULL); - ++(pCache->currentCount); - - pEntry->rid = taosAddRef(pCache->refMgr, pEntry); - ASSERT(pEntry->rid >= 0); - - sNTrace(pCache->pSyncNode, "raft cache add, type:%s,%d, type2:%s,%d, index:%" PRId64 ", bytes:%d", - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType, - pEntry->index, pEntry->bytes); - taosThreadMutexUnlock(&pCache->mutex); - return 1; -} - -// find one, return 1 -// not found, return 0 -// error, return -1 -int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { - ASSERT(ppEntry != NULL); - SSyncRaftEntry* pEntry = NULL; - int32_t code = raftEntryCacheGetEntryP(pCache, index, &pEntry); - if (code == 1) { - int32_t bytes = (int32_t)pEntry->bytes; - *ppEntry = taosMemoryMalloc((int64_t)bytes); - memcpy(*ppEntry, pEntry, pEntry->bytes); - (*ppEntry)->rid = -1; - } else { - *ppEntry = NULL; - } - return code; -} - -// find one, return 1 -// not found, return 0 -// error, return -1 -int32_t raftEntryCacheGetEntryP(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry** ppEntry) { - taosThreadMutexLock(&pCache->mutex); - - SyncIndex index2 = index; - int32_t code = 0; - - SArray* entryPArray = tSkipListGet(pCache->pSkipList, (char*)(&index2)); - int32_t arraySize = taosArrayGetSize(entryPArray); - if (arraySize == 1) { - SSkipListNode** ppNode = (SSkipListNode**)taosArrayGet(entryPArray, 0); - ASSERT(*ppNode != NULL); - *ppEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(*ppNode); - taosAcquireRef(pCache->refMgr, (*ppEntry)->rid); - code = 1; - - } else if (arraySize == 0) { - code = 0; - - } else { - ASSERT(0); - - code = -1; - } - taosArrayDestroy(entryPArray); - - taosThreadMutexUnlock(&pCache->mutex); - return code; -} - -// count = -1, clear all -// count >= 0, clear count -// return -1, error -// return delete count -int32_t raftEntryCacheClear(struct SRaftEntryCache* pCache, int32_t count) { - taosThreadMutexLock(&pCache->mutex); - int32_t returnCnt = 0; - - if (count == -1) { - // clear all - SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); - while (tSkipListIterNext(pIter)) { - SSkipListNode* pNode = tSkipListIterGet(pIter); - ASSERT(pNode != NULL); - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); - syncEntryDestroy(pEntry); - ++returnCnt; - } - tSkipListDestroyIter(pIter); - - tSkipListDestroy(pCache->pSkipList); - pCache->pSkipList = - tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, sizeof(SyncIndex), cmpFn, SL_ALLOW_DUP_KEY, keyFn); - ASSERT(pCache->pSkipList != NULL); - - } else { - // clear count - int i = 0; - SSkipListIterator* pIter = tSkipListCreateIter(pCache->pSkipList); - SArray* delNodeArray = taosArrayInit(0, sizeof(SSkipListNode*)); - - // free entry - while (tSkipListIterNext(pIter)) { - SSkipListNode* pNode = tSkipListIterGet(pIter); - ASSERT(pNode != NULL); - if (i++ >= count) { - break; - } - - // sDebug("push pNode:%p", pNode); - taosArrayPush(delNodeArray, &pNode); - ++returnCnt; - SSyncRaftEntry* pEntry = (SSyncRaftEntry*)SL_GET_NODE_DATA(pNode); - - // syncEntryDestroy(pEntry); - taosRemoveRef(pCache->refMgr, pEntry->rid); - } - tSkipListDestroyIter(pIter); - - // delete skiplist node - int32_t arraySize = taosArrayGetSize(delNodeArray); - for (int32_t i = 0; i < arraySize; ++i) { - SSkipListNode** ppNode = taosArrayGet(delNodeArray, i); - // sDebug("get pNode:%p", *ppNode); - tSkipListRemoveNode(pCache->pSkipList, *ppNode); - } - taosArrayDestroy(delNodeArray); - } - - pCache->currentCount -= returnCnt; - taosThreadMutexUnlock(&pCache->mutex); - return returnCnt; -} diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a83a19928e0a38bdbdd56e9ffc9fdb1927a9ac77..381327d4d7ab57976750ab91be305a8ece0e6ca8 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -168,17 +168,19 @@ static int32_t snapshotSend(SSyncSnapshotSender *pSender) { if (pSender->blockLen > 0) { // has read data - sSDebug(pSender, "snapshot sender continue to read, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); + sSDebug(pSender, "vgId:%d, snapshot sender continue to read, blockLen:%d seq:%d", pSender->pSyncNode->vgId, + pSender->blockLen, pSender->seq); } else { // read finish, update seq to end pSender->seq = SYNC_SNAPSHOT_SEQ_END; - sSInfo(pSender, "snapshot sender read to the end, blockLen:%d seq:%d", pSender->blockLen, pSender->seq); + sSInfo(pSender, "vgId:%d, snapshot sender read to the end, blockLen:%d seq:%d", pSender->pSyncNode->vgId, + pSender->blockLen, pSender->seq); } // build msg SRpcMsg rpcMsg = {0}; if (syncBuildSnapshotSend(&rpcMsg, pSender->blockLen, pSender->pSyncNode->vgId) != 0) { - sSError(pSender, "snapshot sender build msg failed since %s", pSender->pSyncNode->vgId, terrstr()); + sSError(pSender, "vgId:%d, snapshot sender build msg failed since %s", pSender->pSyncNode->vgId, terrstr()); return -1; } @@ -340,11 +342,13 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { taosMemoryFree(pReceiver); } -bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { return pReceiver->start; } +bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver) { + return (pReceiver != NULL ? pReceiver->start : false); +} static int32_t snapshotReceiverStartWriter(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend *pBeginMsg) { if (pReceiver->pWriter != NULL) { - sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null"); + sRError(pReceiver, "vgId:%d, snapshot receiver writer is not null", pReceiver->pSyncNode->vgId); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -851,8 +855,8 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend pMsg->snapBeginIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm); if (pMsg->snapBeginIndex > snapshot.lastApplyIndex) { - sSError(pSender, "prepare snapshot failed since beginIndex:%d larger than applyIndex:%d", pMsg->snapBeginIndex, - snapshot.lastApplyIndex); + sSError(pSender, "prepare snapshot failed since beginIndex:%" PRId64 " larger than applyIndex:%" PRId64, + pMsg->snapBeginIndex, snapshot.lastApplyIndex); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -966,7 +970,8 @@ int32_t syncNodeOnSnapshotRsp(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { if (pSender->pReader == NULL || pSender->finish) { syncLogRecvSyncSnapshotRsp(pSyncNode, pMsg, "snapshot sender invalid"); - sSError(pSender, "snapshot sender invalid, pReader:%p finish:%d", pMsg->code, pSender->pReader, pSender->finish); + sSError(pSender, "snapshot sender invalid error:%s 0x%x, pReader:%p finish:%d", tstrerror(pMsg->code), pMsg->code, + pSender->pReader, pSender->finish); terrno = pMsg->code; goto _ERROR; } diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index bf8c5c53dc99d031ad7e00d9ac4f01b4fd7c1bb3..6df2b4000097a9079da99dedbdb2dca172c37449 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -74,7 +74,10 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg SBTree *pBt; int ret; - ASSERT(keyLen != 0); + if (keyLen == 0) { + tdbError("tdb/btree-open: key len cannot be zero."); + return -1; + } *ppBt = NULL; @@ -152,7 +155,11 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPg tdbPostCommit(pPager->pEnv, txn); } - ASSERT(pgno != 0); + if (pgno == 0) { + tdbError("tdb/btree-open: pgno cannot be zero."); + tdbOsFree(pBt); + return -1; + } pBt->root = pgno; /* // TODO: pBt->root @@ -192,7 +199,7 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-insert: btc move to failed with ret: %d.", ret); return -1; } @@ -202,17 +209,17 @@ int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, in if (c > 0) { btc.idx++; } else if (c == 0) { - // dup key not allowed - tdbError("unable to insert dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); - // ASSERT(0); + // dup key not allowed with insert + tdbBtcClose(&btc); + tdbError("tdb/btree-insert: dup key. pKey: %p, kLen: %d, btc: %p, pTxn: %p", pKey, kLen, &btc, pTxn); return -1; } } ret = tdbBtcUpsert(&btc, pKey, kLen, pVal, vLen, 1); if (ret < 0) { - ASSERT(0); tdbBtcClose(&btc); + tdbError("tdb/btree-insert: btc upsert failed with ret: %d.", ret); return -1; } @@ -233,7 +240,7 @@ int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn) { ret = tdbBtcMoveTo(&btc, pKey, kLen, &c); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-delete: btc move to failed with ret: %d.", ret); return -1; } @@ -316,7 +323,8 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL ret = tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (ret < 0) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: btc move to failed with ret: %d.", ret); + return -1; } if (btc.idx < 0 || cret) { @@ -332,7 +340,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTKey = tdbRealloc(*ppKey, cd.kLen); if (pTKey == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTKey failed."); return -1; } *ppKey = pTKey; @@ -344,7 +352,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL pTVal = tdbRealloc(*ppVal, cd.vLen); if (pTVal == NULL) { tdbBtcClose(&btc); - ASSERT(0); + tdbError("tdb/btree-pget: realloc pTVal failed."); return -1; } *ppVal = pTVal; @@ -357,7 +365,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } if (TDB_CELLDECODER_FREE_VAL(&cd)) { - tdbDebug("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbTrace("tdb btc/pget/2 decoder: %p pVal free: %p", &cd, cd.pVal); tdbFree(cd.pVal); } @@ -373,7 +381,9 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 int mlen; int cret; - ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL); + if (ASSERT(keyLen1 > 0 && keyLen2 > 0 && pKey1 != NULL && pKey2 != NULL)) { + // -1 is less than + } mlen = keyLen1 < keyLen2 ? keyLen1 : keyLen2; cret = memcmp(pKey1, pKey2, mlen); @@ -388,36 +398,7 @@ static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2 } return cret; } -/* -static int tdbBtreeOpenImpl(SBTree *pBt) { - // Try to get the root page of the an existing btree - SPgno pgno; - SPage *pPage; - int ret; - - { - // 1. TODO: Search the main DB to check if the DB exists - ret = tdbPagerOpenDB(pBt->pPager, &pgno, true, pBt); - ASSERT(ret == 0); - } - if (pgno != 0) { - pBt->root = pgno; - return 0; - } - - // Try to create a new database - ret = tdbPagerAllocPage(pBt->pPager, &pgno); - if (ret < 0) { - ASSERT(0); - return -1; - } - - ASSERT(pgno != 0); - pBt->root = pgno; - return 0; -} -*/ int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; @@ -553,11 +534,15 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx nOlds = 3; } for (int i = 0; i < nOlds; i++) { - ASSERT(sIdx + i <= nCells); + if (ASSERT(sIdx + i <= nCells)) { + return -1; + } SPgno pgno; if (sIdx + i == nCells) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent)); + if (ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent))) { + return -1; + } pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -567,7 +552,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx ret = tdbPagerFetchPage(pBt->pPager, &pgno, pOlds + i, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = 0}), pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); return -1; } @@ -686,7 +671,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx szRCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); } - ASSERT(infoNews[iNew - 1].cnt > 0); + if (ASSERT(infoNews[iNew - 1].cnt > 0)) { + return -1; + } if (infoNews[iNew].size + szRCell >= infoNews[iNew - 1].size - szRCell) { break; @@ -729,7 +716,8 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx iarg.flags = flags; ret = tdbPagerFetchPage(pBt->pPager, &pgno, pNews + iNew, tdbBtreeInitPage, &iarg, pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-balance: fetch page failed with ret: %d.", ret); + return -1; } ret = tdbPagerWrite(pBt->pPager, pNews[iNew]); @@ -773,8 +761,12 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx pCell = tdbPageGetCell(pPage, oIdx); szCell = tdbBtreeCellSize(pPage, pCell, 0, NULL, NULL); - ASSERT(nNewCells <= infoNews[iNew].cnt); - ASSERT(iNew < nNews); + if (ASSERT(nNewCells <= infoNews[iNew].cnt)) { + return -1; + } + if (ASSERT(iNew < nNews)) { + return -1; + } if (nNewCells < infoNews[iNew].cnt) { tdbPageInsertCell(pNews[iNew], nNewCells, pCell, szCell, 0); @@ -813,14 +805,20 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } } } else { - ASSERT(childNotLeaf); - ASSERT(iNew < nNews - 1); + if (ASSERT(childNotLeaf)) { + return -1; + } + if (ASSERT(iNew < nNews - 1)) { + return -1; + } // set current new page right-most child ((SIntHdr *)pNews[iNew]->pData)->pgno = ((SPgno *)pCell)[0]; // insert to parent as divider cell - ASSERT(iNew < nNews - 1); + if (ASSERT(iNew < nNews - 1)) { + return -1; + } ((SPgno *)pCell)[0] = TDB_PAGE_PGNO(pNews[iNew]); tdbPageInsertCell(pParent, sIdx++, pCell, szCell, 0); @@ -835,7 +833,9 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx } if (childNotLeaf) { - ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt); + if (ASSERT(TDB_PAGE_TOTAL_CELLS(pNews[nNews - 1]) == infoNews[nNews - 1].cnt)) { + return -1; + } ((SIntHdr *)(pNews[nNews - 1]->pData))->pgno = rPgno; SIntHdr *pIntHdr = (SIntHdr *)pParent->pData; @@ -1025,7 +1025,9 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= kLen; // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { - ASSERT(pVal != NULL && vLen != 0); + if (ASSERT(pVal != NULL && vLen != 0)) { + return -1; + } memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); nLeft -= nLocal - nHeader - kLen - sizeof(SPgno); } @@ -1190,9 +1192,15 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo int nPayload; int ret; - ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); - ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); - ASSERT(pKey != NULL && kLen > 0); + if (ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen)) { + return -1; + } + if (ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen)) { + return -1; + } + if (ASSERT(pKey != NULL && kLen > 0)) { + return -1; + } nPayload = 0; nHeader = 0; @@ -1201,7 +1209,10 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo // 1. Encode Header part /* Encode SPgno if interior page */ if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-encode-cell: invalid cell."); + return -1; + } ((SPgno *)(pCell + nHeader))[0] = ((SPgno *)pVal)[0]; nHeader = nHeader + sizeof(SPgno); @@ -1226,8 +1237,8 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo ret = tdbBtreeEncodePayload(pPage, pCell, nHeader, pKey, kLen, pVal, vLen, &nPayload, pTxn, pBt); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btree-encode-cell: encode payload failed with ret: %d.", ret); + return -1; } *szCell = nHeader + nPayload; @@ -1244,7 +1255,10 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int vLen = pDecoder->vLen; if (pDecoder->pVal) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btree-decode-payload: leaf page with non-null pVal."); + return -1; + } nPayload = pDecoder->kLen; } else { nPayload = pDecoder->kLen + pDecoder->vLen; @@ -1451,7 +1465,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD // 1. Decode header part if (!leaf) { - ASSERT(pPage->vLen == sizeof(SPgno)); + if (pPage->vLen != sizeof(SPgno)) { + tdbError("tdb/btree-decode-cell: invalid cell."); + return -1; + } pDecoder->pgno = ((SPgno *)(pCell + nHeader))[0]; pDecoder->pVal = (u8 *)(&(pDecoder->pgno)); @@ -1465,7 +1482,10 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-decode-cell: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); } else { pDecoder->vLen = pPage->vLen; @@ -1497,7 +1517,10 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * } if (pPage->vLen == TDB_VARIANT_LEN) { - ASSERT(leaf); + if (!leaf) { + tdbError("tdb/btree-cell-size: not a leaf page."); + return -1; + } nHeader += tdbGetVarInt(pCell + nHeader, &vLen); } else if (leaf) { vLen = pPage->vLen; @@ -1593,29 +1616,42 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: fetch page failed with ret: %d.", ret); return -1; } - ASSERT(TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)) { + tdbError("tdb/btc-move-tofirst: not a root page"); + return -1; + } pBtc->iPage = 0; if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0) { pBtc->idx = 0; } else { // no any data, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-first: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-first: move from a dirty cursor."); + return -1; #if 0 // move from a position int iPage = 0; for (; iPage < pBtc->iPage; iPage++) { - ASSERT(pBtc->idxStack[iPage] >= 0); + if (pBtc->idxStack[iPage] < 0) { + tdbError("tdb/btc-move-to-first: invalid idx: %d.", pBtc->idxStack[iPage]); + return -1; + } + if (pBtc->idxStack[iPage]) break; } @@ -1637,7 +1673,7 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tofirst: btc move downward failed with ret: %d.", ret); return -1; } @@ -1662,7 +1698,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbPagerFetchPage(pPager, &pBt->root, &(pBtc->pPage), tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: fetch page failed with ret: %d.", ret); return -1; } @@ -1672,18 +1708,28 @@ int tdbBtcMoveToLast(SBTC *pBtc) { pBtc->idx = TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage) ? nCells - 1 : nCells; } else { // no data at all, point to an invalid position - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-last: not a leaf page."); + return -1; + } + pBtc->idx = -1; return 0; } } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to-last: move from a dirty cursor."); + return -1; #if 0 int iPage = 0; // downward search for (; iPage < pBtc->iPage; iPage++) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pgStack[iPage])) { + tdbError("tdb/btc-move-to-last: leaf page in cursor stack."); + return -1; + } + nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pgStack[iPage]); if (pBtc->idxStack[iPage] != nCells) break; } @@ -1710,7 +1756,7 @@ int tdbBtcMoveToLast(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tolast: btc move downward failed with ret: %d.", ret); return -1; } @@ -1772,7 +1818,7 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToNext(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-next: btc move to next failed with ret: %d.", ret); return -1; } @@ -1818,7 +1864,7 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { ret = tdbBtcMoveToPrev(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btree-prev: btc move to prev failed with ret: %d.", ret); return -1; } @@ -1830,7 +1876,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { int ret; SCell *pCell; - ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-to-next: not a leaf page."); + return -1; + } if (pBtc->idx < 0) return -1; @@ -1849,7 +1898,10 @@ int tdbBtcMoveToNext(SBTC *pBtc) { tdbBtcMoveUpward(pBtc); pBtc->idx++; - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btree-decode-cell: should not be a leaf page here."); + return -1; + } if (pBtc->idx <= TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { break; } @@ -1861,7 +1913,7 @@ int tdbBtcMoveToNext(SBTC *pBtc) { ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-tonext: btc move downward failed with ret: %d.", ret); return -1; } @@ -1913,8 +1965,15 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { SPgno pgno; SCell *pCell; - ASSERT(pBtc->idx >= 0); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + if (pBtc->idx < 0) { + tdbError("tdb/btc-move-downward: invalid idx: %d.", pBtc->idx); + return -1; + } + + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { + tdbError("tdb/btc-move-downward: should not be a leaf page here."); + return -1; + } if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); @@ -1923,7 +1982,10 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; } - ASSERT(pgno); + if (!pgno) { + tdbError("tdb/btc-move-downward: invalid pgno."); + return -1; + } pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; @@ -1934,7 +1996,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc) { ret = tdbPagerFetchPage(pBtc->pBt->pPager, &pgno, &pBtc->pPage, tdbBtreeInitPage, &((SBtreeInitPageArg){.pBt = pBtc->pBt, .flags = 0}), pBtc->pTxn); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-move-downward: fetch page failed with ret: %d.", ret); return -1; } @@ -1989,7 +2051,10 @@ int tdbBtcDelete(SBTC *pBtc) { int nKey; int ret; - ASSERT(idx >= 0 && idx < nCells); + if (idx < 0 || idx >= nCells) { + tdbError("tdb/btc-delete: idx: %d out of range[%d, %d).", idx, 0, nCells); + return -1; + } // drop the cell on the leaf ret = tdbPagerWrite(pPager, pBtc->pPage); @@ -2027,7 +2092,7 @@ int tdbBtcDelete(SBTC *pBtc) { ret = tdbPageUpdateCell(pPage, idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { tdbOsFree(pCell); - ASSERT(0); + tdbError("tdb/btc-delete: page update cell failed with ret: %d.", ret); return -1; } tdbOsFree(pCell); @@ -2038,11 +2103,14 @@ int tdbBtcDelete(SBTC *pBtc) { } } else { // delete the leaf page and do balance - ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0); + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) != 0) { + tdbError("tdb/btc-delete: page to be deleted should be empty."); + return -1; + } ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-delete: btree balance failed with ret: %d.", ret); return -1; } } @@ -2059,13 +2127,16 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int void *pBuf; int ret; - ASSERT(pBtc->idx >= 0); + if (pBtc->idx < 0) { + tdbError("tdb/btc-upsert: invalid idx: %d.", pBtc->idx); + return -1; + } // alloc space szBuf = kLen + nData + 14; pBuf = tdbRealloc(pBtc->pBt->pBuf, pBtc->pBt->pageSize > szBuf ? szBuf : pBtc->pBt->pageSize); if (pBuf == NULL) { - ASSERT(0); + tdbError("tdb/btc-upsert: realloc pBuf failed."); return -1; } pBtc->pBt->pBuf = pBuf; @@ -2074,7 +2145,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // encode cell ret = tdbBtreeEncodeCell(pBtc->pPage, pKey, kLen, pData, nData, pCell, &szCell, pBtc->pTxn, pBtc->pBt); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree encode cell failed with ret: %d.", ret); return -1; } @@ -2087,16 +2158,22 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int // insert or update if (insert) { - ASSERT(pBtc->idx <= nCells); + if (pBtc->idx > nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageInsertCell(pBtc->pPage, pBtc->idx, pCell, szCell, 0); } else { - ASSERT(pBtc->idx < nCells); + if (pBtc->idx >= nCells) { + tdbError("tdb/btc-upsert: invalid idx: %d, nCells: %d.", pBtc->idx, nCells); + return -1; + } ret = tdbPageUpdateCell(pBtc->pPage, pBtc->idx, pCell, szCell, pBtc->pTxn, pBtc->pBt); } if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: page insert/update cell failed with ret: %d.", ret); return -1; } @@ -2104,7 +2181,7 @@ int tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int if (pBtc->pPage->nOverflow > 0) { ret = tdbBtreeBalance(pBtc); if (ret < 0) { - ASSERT(0); + tdbError("tdb/btc-upsert: btree balance failed with ret: %d.", ret); return -1; } } @@ -2129,8 +2206,8 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { &((SBtreeInitPageArg){.pBt = pBt, .flags = TDB_BTREE_ROOT | TDB_BTREE_LEAF}), pBtc->pTxn); if (ret < 0) { // TODO - ASSERT(0); - return 0; + tdbError("tdb/btc-move-to: fetch page failed with ret: %d.", ret); + return -1; } pBtc->iPage = 0; @@ -2138,7 +2215,9 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // for empty tree, just return with an invalid position if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) return 0; } else { - ASSERT(0); + // TODO + tdbError("tdb/btc-move-to: move from a dirty cursor."); + return -1; #if 0 SPage *pPage; int idx; @@ -2150,7 +2229,10 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { idx = pBtc->idxStack[iPage]; nCells = TDB_PAGE_TOTAL_CELLS(pPage); - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); + if (TDB_BTREE_PAGE_IS_LEAF(pPage)) { + tdbError("tdb/btc-move-to: leaf page in cursor stack."); + return -1; + } // check if key <= current position if (idx < nCells) { @@ -2252,7 +2334,10 @@ int tdbBtcClose(SBTC *pBtc) { if (pBtc->iPage < 0) return 0; for (;;) { - ASSERT(pBtc->pPage); + if (NULL == pBtc->pPage) { + tdbError("tdb/btc-close: null ptr pPage."); + return -1; + } tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage, pBtc->pTxn); diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index c79279c658a746e83a40eaed6fd6cb8cbe6e31d7..952c49db73a2c8986d67c8e327175ad00e778ae6 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -247,7 +247,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { // remove from the list for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // remove from hash @@ -255,7 +258,10 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { ppPager = &pDb->pgrHash[hash % pDb->nPgrHash]; for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) { } - ASSERT(*ppPager == pPager); + if (*ppPager != pPager) { + tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager); + return; + } *ppPager = pPager->pNext; // decrease the counter diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 4896568c7f5ce45ce816f16d3eec0436cf51dfc5..262f3d27e648cfebb4f081c725f443756cb6a06d 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -236,10 +236,10 @@ void tdbPCacheInvalidatePage(SPCache *pCache, SPager *pPager, SPgno pgno) { void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { i32 nRef; - ASSERT(pTxn); - - // nRef = tdbUnrefPage(pPage); - // ASSERT(nRef >= 0); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, release failed."); + return; + } tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); @@ -275,7 +275,10 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage = NULL; SPage *pPageH = NULL; - ASSERT(pTxn); + if (!pTxn) { + tdbError("tdb/pcache: null ptr pTxn, fetch impl failed."); + return NULL; + } // 1. Search the hash table pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; @@ -315,8 +318,8 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) if (!pPage && pTxn->xMalloc != NULL) { ret = tdbPageCreate(pCache->szPage, &pPage, pTxn->xMalloc, pTxn->xArg); if (ret < 0 || pPage == NULL) { - // TODO - ASSERT(0); + tdbError("tdb/pcache: ret: %" PRId32 " pPage: %p, page create failed.", ret, pPage); + // TODO: recycle other backup pages return NULL; } @@ -370,7 +373,11 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { if (pPage->pLruNext != NULL) { - ASSERT(tdbGetPageRef(pPage) == 0); + int32_t nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: pin page's ref not zero: %" PRId32, nRef); + return; + } pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; @@ -383,13 +390,23 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { } static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { - i32 nRef; - - ASSERT(pPage->isLocal); - ASSERT(!pPage->isDirty); - ASSERT(tdbGetPageRef(pPage) == 0); - - ASSERT(pPage->pLruNext == NULL); + i32 nRef = tdbGetPageRef(pPage); + if (nRef != 0) { + tdbError("tdb/pcache: unpin page's ref not zero: %" PRId32, nRef); + return; + } + if (!pPage->isLocal) { + tdbError("tdb/pcache: unpin page's not local: %" PRIu8, pPage->isLocal); + return; + } + if (pPage->isDirty) { + tdbError("tdb/pcache: unpin page's dirty: %" PRIu8, pPage->isDirty); + return; + } + if (NULL != pPage->pLruNext) { + tdbError("tdb/pcache: unpin page's pLruNext not null."); + return; + } tdbTrace("pCache:%p unpin page %p/%d, nPages:%d, pgno:%d, ", pCache, pPage, pPage->id, pCache->nPages, TDB_PAGE_PGNO(pPage)); diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index dc448c986e628b4c7af9dd03aa84706c8053da71..c6310f4985da8fc18a4502d5a4dda5ecfa0d8a64 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -43,9 +43,15 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) u8 *ptr; int size; - ASSERT(xMalloc); + if (!xMalloc) { + tdbError("tdb/page-create: null xMalloc."); + return -1; + } - ASSERT(TDB_IS_PGSIZE_VLD(pageSize)); + if (!TDB_IS_PGSIZE_VLD(pageSize)) { + tdbError("tdb/page-create: invalid pageSize: %d.", pageSize); + return -1; + } *ppPage = NULL; size = pageSize + sizeof(*pPage); @@ -69,16 +75,24 @@ int tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t) *ppPage = pPage; - tdbTrace("page/create: %p/%d %p", pPage, pPage->id, xMalloc); + tdbTrace("tdb/page-create: %p/%d %p", pPage, pPage->id, xMalloc); return 0; } int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) { u8 *ptr; - tdbTrace("page/destroy: %p/%d %p", pPage, pPage->id, xFree); - ASSERT(!pPage->isDirty); - ASSERT(xFree); + tdbTrace("tdb/page-destroy: %p/%d %p", pPage, pPage->id, xFree); + + if (pPage->isDirty) { + tdbError("tdb/page-destroy: dirty page: %" PRIu8 ".", pPage->isDirty); + return -1; + } + + if (!xFree) { + tdbError("tdb/page-destroy: null xFree."); + return -1; + } for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); @@ -105,7 +119,10 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT((u8 *)pPage->pPageFtr == pPage->pFreeEnd); + if ((u8 *)pPage->pPageFtr != pPage->pFreeEnd) { + tdbError("tdb/page-zero: invalid page, pFreeEnd: %p, pPageFtr: %p", pPage->pFreeEnd, pPage->pPageFtr); + return; + } } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { @@ -121,8 +138,15 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell pPage->nOverflow = 0; pPage->xCellSize = xCellSize; - ASSERT(pPage->pFreeEnd >= pPage->pFreeStart); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart <= TDB_PAGE_NFREE(pPage)); + if (pPage->pFreeEnd < pPage->pFreeStart) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p", pPage->pFreeEnd, pPage->pFreeStart); + return; + } + if (pPage->pFreeEnd - pPage->pFreeStart > TDB_PAGE_NFREE(pPage)) { + tdbError("tdb/page-init: invalid page, pFreeEnd: %p, pFreeStart: %p, NFREE: %d", pPage->pFreeEnd, pPage->pFreeStart, + TDB_PAGE_NFREE(pPage)); + return; + } } int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl) { @@ -132,7 +156,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl int lidx; // local idx SCell *pNewCell; - ASSERT(szCell <= TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + if (szCell > TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)) { + tdbError("tdb/page-insert-cell: invalid page, szCell: %d, max free: %lu", szCell, + TDB_PAGE_MAX_FREE_BLOCK(pPage, pPage->pPageHdr - pPage->pData)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); nCells = TDB_PAGE_NCELLS(pPage); @@ -176,7 +204,11 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl TDB_PAGE_CELL_OFFSET_AT_SET(pPage, lidx, pNewCell - pPage->pData); TDB_PAGE_NCELLS_SET(pPage, nCells + 1); - ASSERT(pPage->pFreeStart == pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)); + if (pPage->pFreeStart != pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * (nCells + 1)) { + tdbError("tdb/page-insert-cell: invalid page, pFreeStart: %p, pCellIdx: %p, nCells: %d", pPage->pFreeStart, + pPage->pCellIdx, nCells); + return -1; + } } for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -200,7 +232,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { nCells = TDB_PAGE_NCELLS(pPage); - ASSERT(idx >= 0 && idx < nCells + pPage->nOverflow); + if (idx < 0 || idx >= nCells + pPage->nOverflow) { + tdbError("tdb/page-drop-cell: idx: %d out of range, nCells: %d, nOvfl: %d.", idx, nCells, pPage->nOverflow); + return -1; + } iOvfl = 0; for (; iOvfl < pPage->nOverflow; iOvfl++) { @@ -228,7 +263,10 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { for (; iOvfl < pPage->nOverflow; iOvfl++) { pPage->aiOvfl[iOvfl]--; - ASSERT(pPage->aiOvfl[iOvfl] > 0); + if (pPage->aiOvfl[iOvfl] <= 0) { + tdbError("tdb/page-drop-cell: invalid ai idx: %d", pPage->aiOvfl[iOvfl]); + return -1; + } } return 0; @@ -240,12 +278,19 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); pToPage->pFreeEnd = (u8 *)(pToPage->pPageFtr) - ((u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(pToPage->pFreeEnd >= pToPage->pFreeStart); + if (pToPage->pFreeEnd < pToPage->pFreeStart) { + tdbError("tdb/page-copy: invalid to page, pFreeStart: %p, pFreeEnd: %p", pToPage->pFreeStart, pToPage->pFreeEnd); + return; + } memcpy(pToPage->pPageHdr, pFromPage->pPageHdr, pFromPage->pFreeStart - pFromPage->pPageHdr); memcpy(pToPage->pFreeEnd, pFromPage->pFreeEnd, (u8 *)pFromPage->pPageFtr - pFromPage->pFreeEnd); - ASSERT(TDB_PAGE_CCELLS(pToPage) == pToPage->pFreeEnd - pToPage->pData); + if (TDB_PAGE_CCELLS(pToPage) != pToPage->pFreeEnd - pToPage->pData) { + tdbError("tdb/page-copy: invalid to page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pToPage), + pToPage->pFreeEnd - pToPage->pData); + return; + } delta = (pToPage->pPageHdr - pToPage->pData) - (pFromPage->pPageHdr - pFromPage->pData); if (delta != 0) { @@ -295,8 +340,16 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { *ppCell = NULL; nFree = TDB_PAGE_NFREE(pPage); - ASSERT(nFree >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)); - ASSERT(TDB_PAGE_CCELLS(pPage) == pPage->pFreeEnd - pPage->pData); + if (nFree < szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { + tdbError("tdb/page-allocate: invalid cell size, nFree: %d, szCell: %d, szOffset: %d", nFree, szCell, + TDB_PAGE_OFFSET_SIZE(pPage)); + return -1; + } + if (TDB_PAGE_CCELLS(pPage) != pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: invalid page, cell body: %d, range: %ld", TDB_PAGE_CCELLS(pPage), + pPage->pFreeEnd - pPage->pData); + return -1; + } // 1. Try to allocate from the free space block area if (pPage->pFreeEnd - pPage->pFreeStart >= szCell + TDB_PAGE_OFFSET_SIZE(pPage)) { @@ -308,7 +361,10 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 2. Try to allocate from the page free list cellFree = TDB_PAGE_FCELL(pPage); - ASSERT(cellFree == 0 || cellFree >= pPage->pFreeEnd - pPage->pData); + if (cellFree != 0 && cellFree < pPage->pFreeEnd - pPage->pData) { + tdbError("tdb/page-allocate: cellFree: %d, pFreeEnd: %p, pData: %p.", cellFree, pPage->pFreeEnd, pPage->pData); + return -1; + } if (cellFree && pPage->pFreeEnd - pPage->pFreeStart >= TDB_PAGE_OFFSET_SIZE(pPage)) { SCell *pPrevFreeCell = NULL; int szPrevFreeCell; @@ -353,16 +409,30 @@ static int tdbPageAllocate(SPage *pPage, int szCell, SCell **ppCell) { // 3. Try to dfragment and allocate again tdbPageDefragment(pPage); - ASSERT(pPage->pFreeEnd - pPage->pFreeStart == nFree); - ASSERT(nFree == TDB_PAGE_NFREE(pPage)); - ASSERT(pPage->pFreeEnd - pPage->pData == TDB_PAGE_CCELLS(pPage)); + if (pPage->pFreeEnd - pPage->pFreeStart != nFree) { + tdbError("tdb/page-allocate: nFree: %d, pFreeStart: %p, pFreeEnd: %p.", nFree, pPage->pFreeStart, pPage->pFreeEnd); + return -1; + } + if (TDB_PAGE_NFREE(pPage) != nFree) { + tdbError("tdb/page-allocate: nFree: %d, page free: %d.", nFree, TDB_PAGE_NFREE(pPage)); + return -1; + } + if (pPage->pFreeEnd - pPage->pData != TDB_PAGE_CCELLS(pPage)) { + tdbError("tdb/page-allocate: ccells: %d, pFreeStart: %p, pData: %p.", TDB_PAGE_CCELLS(pPage), pPage->pFreeStart, + pPage->pData); + return -1; + } pPage->pFreeEnd -= szCell; pCell = pPage->pFreeEnd; TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData); _alloc_finish: - ASSERT(pCell); + if (NULL == pCell) { + tdbError("tdb/page-allocate: null ptr pCell."); + return -1; + } + pPage->pFreeStart += TDB_PAGE_OFFSET_SIZE(pPage); TDB_PAGE_NFREE_SET(pPage, nFree - szCell - TDB_PAGE_OFFSET_SIZE(pPage)); *ppCell = pCell; @@ -375,9 +445,18 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { u8 *dest; u8 *src; - ASSERT(pCell >= pPage->pFreeEnd); - ASSERT(pCell + szCell <= (u8 *)(pPage->pPageFtr)); - ASSERT(pCell == TDB_PAGE_CELL_AT(pPage, idx)); + if (pCell < pPage->pFreeEnd) { + tdbError("tdb/page-free: invalid cell, cell: %p, free end: %p", pCell, pPage->pFreeEnd); + return -1; + } + if (pCell + szCell > (u8 *)(pPage->pPageFtr)) { + tdbError("tdb/page-free: cell crosses page footer, cell: %p, size: %d footer: %p", pCell, szCell, pPage->pFreeEnd); + return -1; + } + if (pCell != TDB_PAGE_CELL_AT(pPage, idx)) { + tdbError("tdb/page-free: cell pos incorrect, cell: %p, pos: %p", pCell, TDB_PAGE_CELL_AT(pPage, idx)); + return -1; + } nFree = TDB_PAGE_NFREE(pPage); @@ -390,7 +469,8 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) { pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree); TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData); } else { - ASSERT(0); + tdbError("tdb/page-free: invalid cell size: %d", szCell); + return -1; } } @@ -471,39 +551,59 @@ typedef struct { // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } static inline void setPageCellNum(SPage *pPage, int cellNum) { - ASSERT(cellNum < 65536); + if (cellNum >= 65536) { + tdbError("tdb/page-set-cell-num: invalid cellNum: %d.", cellNum); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellNum = (u16)cellNum; } // cellBody static inline int getPageCellBody(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellBody; } static inline void setPageCellBody(SPage *pPage, int cellBody) { - ASSERT(cellBody < 65536); + if (cellBody >= 65536) { + tdbError("tdb/page-set-cell-body: invalid cellBody: %d.", cellBody); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellBody = (u16)cellBody; } // cellFree static inline int getPageCellFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellFree; } static inline void setPageCellFree(SPage *pPage, int cellFree) { - ASSERT(cellFree < 65536); + if (cellFree >= 65536) { + tdbError("tdb/page-set-cell-free: invalid cellFree: %d.", cellFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].cellFree = (u16)cellFree; } // nFree static inline int getPageNFree(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].nFree; } static inline void setPageNFree(SPage *pPage, int nFree) { - ASSERT(nFree < 65536); + if (nFree >= 65536) { + tdbError("tdb/page-set-nfree: invalid nFree: %d.", nFree); + return; + } ((SPageHdr *)(pPage->pPageHdr))[0].nFree = (u16)nFree; } // cell offset static inline int getPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getPageCellNum(pPage)); + int cellNum = getPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/page-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return ((u16 *)pPage->pCellIdx)[idx]; } static inline void setPageCellOffset(SPage *pPage, int idx, int offset) { - ASSERT(offset < 65536); + if (offset >= 65536) { + tdbError("tdb/page-set-cell-offset: invalid offset: %d.", offset); + return; + } ((u16 *)pPage->pCellIdx)[idx] = (u16)offset; } @@ -578,7 +678,12 @@ static inline void setLPageNFree(SPage *pPage, int nFree) { // cell offset static inline int getLPageCellOffset(SPage *pPage, int idx) { - ASSERT(idx >= 0 && idx < getLPageCellNum(pPage)); + int cellNum = getLPageCellNum(pPage); + if (idx < 0 || idx >= cellNum) { + tdbError("tdb/lpage-cell-offset: idx: %d out of range[%d, %d).", idx, 0, cellNum); + return -1; + } + return TDB_GET_U24(pPage->pCellIdx + 3 * idx); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 63e88b0a12854780a7c153c7bffcae287dd987bf..29b7fa740c85e9ea74def8e53d47edd69c76b743 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -14,7 +14,7 @@ */ #include "tdbInt.h" - +/* #pragma pack(push, 1) typedef struct { u8 hdrString[16]; @@ -26,7 +26,7 @@ typedef struct { #pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); - +*/ struct hashset_st { size_t nbits; size_t mask; @@ -234,7 +234,6 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { int ret; SPage **ppPage; - // ASSERT(pPager->inTran); if (pPage->isDirty) return 0; // ref page one more time so the page will not be release @@ -243,23 +242,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Set page as dirty pPage->isDirty = 1; - /* - // Add page to dirty list(TODO: NOT use O(n^2) algorithm) - for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); - ppPage = &((*ppPage)->pDirtyNext)) { - } - - if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) { - tdbUnrefPage(pPage); - return 0; - } - - ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); - pPage->pDirtyNext = *ppPage; - *ppPage = pPage; - */ - tdbTrace("put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); + tdbTrace("tdb/pager-write: put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt); tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary @@ -327,7 +311,11 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { while ((pNode = tRBTreeIterNext(&iter)) != NULL) { pPage = (SPage *)pNode; - ASSERT(pPage->nOverflow == 0); + if (pPage->nOverflow != 0) { + tdbError("tdb/pager-commit: %p, pPage: %p, ovfl: %d, commit page failed.", pPager, pPage, pPage->nOverflow); + return -1; + } + ret = tdbPagerPWritePageToDB(pPager, pPage); if (ret < 0) { tdbError("failed to write page to db since %s", tstrerror(terrno)); @@ -657,12 +645,15 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa loadPage = 0; ret = tdbPagerAllocPage(pPager, &pgno); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); return -1; } } - ASSERT(pgno > 0); + if (pgno == 0) { + tdbError("tdb/pager: %p, ret: %d pgno: %" PRIu32 ", alloc page failed.", pPager, ret, pgno); + return -1; + } // fetch a page container memcpy(&pgid, pPager->fid, TDB_FILE_ID_LEN); @@ -676,7 +667,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa if (!TDB_PAGE_INITIALIZED(pPage)) { ret = tdbPagerInitPage(pPager, pPage, initPage, arg, loadPage); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager: %p, pPage: %p, init page failed.", pPager, pPage); return -1; } } @@ -684,8 +675,14 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPa // printf("thread %" PRId64 " pager fetch page %d pgno %d ppage %p\n", taosGetSelfPthreadId(), pPage->id, // TDB_PAGE_PGNO(pPage), pPage); - ASSERT(TDB_PAGE_INITIALIZED(pPage)); - ASSERT(pPage->pPager == pPager); + if (!TDB_PAGE_INITIALIZED(pPage)) { + tdbError("tdb/pager: %p, pPage: %p, fetch page uninited.", pPager, pPage); + return -1; + } + if (pPage->pPager != pPager) { + tdbError("tdb/pager: %p/%p, fetch page failed.", pPager, pPage->pPager); + return -1; + } *ppgno = pgno; *ppPage = pPage; @@ -727,8 +724,10 @@ int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno) { return -1; } - ASSERT(*ppgno != 0); - + if (*ppgno == 0) { + tdbError("tdb/pager:%p, alloc new page failed.", pPager); + return -1; + } return 0; } @@ -737,8 +736,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage int ret; int lcode; int nLoops; - i64 nRead; - SPgno pgno; + i64 nRead = 0; + SPgno pgno = 0; int init = 0; lcode = TDB_TRY_LOCK_PAGE(pPage); @@ -757,7 +756,6 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage nRead = tdbOsPRead(pPager->fd, pPage->pData, pPage->pageSize, ((i64)pPage->pageSize) * (pgno - 1)); tdbTrace("tdb/pager:%p, pgno:%d, nRead:%" PRId64, pPager, pgno, nRead); if (nRead < pPage->pageSize) { - ASSERT(0); tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32, pPager, pgno, nRead, pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; @@ -768,7 +766,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage ret = (*initPage)(pPage, arg, init); if (ret < 0) { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " init page failed.", pPager, pgno, nRead, + pPage->pageSize); TDB_UNLOCK_PAGE(pPage); return -1; } @@ -787,7 +786,8 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage } } } else { - ASSERT(0); + tdbError("tdb/pager:%p, pgno:%d, nRead:%" PRId64 "pgSize:%" PRId32 " lock page failed.", pPager, pgno, nRead, + pPage->pageSize); return -1; } diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 972e2f29e545ae24c9fd4dd19ec29f553480c602..18d14fa474f51104442cf689bd52649598b9c396 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -105,8 +105,6 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF #endif - ASSERT(pPager != NULL); - if (rollback) { ret = tdbPagerRestoreJournals(pPager); if (ret < 0) { diff --git a/source/libs/tdb/src/db/tdbTxn.c b/source/libs/tdb/src/db/tdbTxn.c index 24f955fe2fde29c089b4768baae54eaea892d09e..0aeed3c140f8317c6abcec16a9fd049f6be89e1a 100644 --- a/source/libs/tdb/src/db/tdbTxn.c +++ b/source/libs/tdb/src/db/tdbTxn.c @@ -18,7 +18,10 @@ int tdbTxnOpen(TXN *pTxn, int64_t txnid, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg, int flags) { // not support read-committed version at the moment - ASSERT(flags == 0 || flags == (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)); + if (flags != 0 && flags != (TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED)) { + tdbError("tdb/txn: invalid txn flags: %" PRId32, flags); + return -1; + } pTxn->flags = flags; pTxn->txnId = txnid; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 6c96bd1dcb44c62073ab37a7725cfc83932167cd..86b36b9b12b74e0d4bb0acf9676c67c258c3062f 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -240,7 +240,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { if (tfsMkdirAt(pTfs, rname, diskId) < 0) { if (errno == ENOENT) { // Try to create upper - char *s = strdup(rname); + char *s = taosStrdup(rname); // Make a copy of dirname(s) because the implementation of 'dirname' differs on different platforms. // Some platform may modify the contents of the string passed into dirname(). Others may return a pointer to @@ -248,7 +248,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) { // the pointer directly in this recursion. // See // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html - char *dir = strdup(taosDirName(s)); + char *dir = taosStrdup(taosDirName(s)); if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) { taosMemoryFree(s); @@ -284,7 +284,9 @@ int32_t tfsMkdir(STfs *pTfs, const char *rname) { } int32_t tfsRmdir(STfs *pTfs, const char *rname) { - ASSERT(rname[0] != 0); + if (rname[0] == 0) { + return 0; + } char aname[TMPNAME_LEN] = "\0"; @@ -293,7 +295,7 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { for (int32_t id = 0; id < pTier->ndisk; id++) { STfsDisk *pDisk = pTier->disks[id]; snprintf(aname, TMPNAME_LEN, "%s%s%s", pDisk->path, TD_DIRSEP, rname); - uInfo("tfs remove dir : path:%s aname:%s rname:[%s]", pDisk->path, aname, rname); + uInfo("tfs remove dir:%s aname:%s rname:[%s]", pDisk->path, aname, rname); taosRemoveDir(aname); } } @@ -301,7 +303,7 @@ int32_t tfsRmdir(STfs *pTfs, const char *rname) { return 0; } -int32_t tfsRename(STfs *pTfs, char *orname, char *nrname) { +int32_t tfsRename(STfs *pTfs, const char *orname, const char *nrname) { char oaname[TMPNAME_LEN] = "\0"; char naname[TMPNAME_LEN] = "\0"; diff --git a/source/libs/tfs/src/tfsDisk.c b/source/libs/tfs/src/tfsDisk.c index ff40529ab2d75ec90be9b35ad259594f869d048e..3717bf1a6dee9deca0df7242ccb9f0c8a70a0e08 100644 --- a/source/libs/tfs/src/tfsDisk.c +++ b/source/libs/tfs/src/tfsDisk.c @@ -23,7 +23,7 @@ STfsDisk *tfsNewDisk(int32_t level, int32_t id, const char *path) { return NULL; } - pDisk->path = strdup(path); + pDisk->path = taosStrdup(path); if (pDisk->path == NULL) { taosMemoryFree(pDisk); terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 8e5f79137fd5f693e012849ce468fffed2870fa9..bab86948fae5e718c39d7090690f8833c80f30c1 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -298,8 +298,8 @@ int32_t httpSendQuit() { static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); - msg->server = strdup(server); - msg->uri = strdup(uri); + msg->server = taosStrdup(server); + msg->uri = taosStrdup(uri); msg->port = port; msg->cont = taosMemoryMalloc(contLen); memcpy(msg->cont, pCont, contLen); diff --git a/source/libs/transport/src/tmsgcb.c b/source/libs/transport/src/tmsgcb.c index af2528bc9247adab655227bc8761f66ed1678491..fc95cab76e867817e53c4bde6b6507e9472a40f5 100644 --- a/source/libs/transport/src/tmsgcb.c +++ b/source/libs/transport/src/tmsgcb.c @@ -59,12 +59,16 @@ void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.re void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); } -void tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) { - (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port); +bool tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) { + if (defaultMsgCb.updateDnodeInfoFp) { + return (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port); + } else { + return false; + } } void tmsgUpdateDnodeEpSet(SEpSet* epset) { for (int32_t i = 0; i < epset->numOfEps; ++i) { tmsgUpdateDnodeInfo(NULL, NULL, epset->eps[i].fqdn, &epset->eps[i].port); } -} \ No newline at end of file +} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 38189f90dbb3e3da63a5ea25ed6c2546390de6f7..e008320222e2c9f2ace809a1a6de919a90b33677 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1023,7 +1023,7 @@ static void cliHandleBatchReq(SCliBatch* pBatch, SCliThrd* pThrd) { if (conn == NULL) { conn = cliCreateConn(pThrd); conn->pBatch = pBatch; - conn->ip = strdup(pList->dst); + conn->ip = taosStrdup(pList->dst); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, pList->ip); if (ipaddr == 0xffffffff) { @@ -1406,7 +1406,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { uint16_t port = EPSET_GET_INUSE_PORT(&pCtx->epSet); CONN_CONSTRUCT_HASH_KEY(key, fqdn, port); - conn->ip = strdup(key); + conn->ip = taosStrdup(key); uint32_t ipaddr = cliGetIpFromFqdnCache(pThrd->fqdn2ipCache, fqdn); if (ipaddr == 0xffffffff) { @@ -1529,8 +1529,8 @@ static void cliBatchDealReq(queue* wq, SCliThrd* pThrd) { pBatchList->batchLenLimit = pInst->batchSize; pBatchList->len += 1; - pBatchList->ip = strdup(ip); - pBatchList->dst = strdup(key); + pBatchList->ip = taosStrdup(ip); + pBatchList->dst = taosStrdup(key); pBatchList->port = port; SCliBatch* pBatch = taosMemoryCalloc(1, sizeof(SCliBatch)); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 707c47f6b17b479b731b2481100fefc66671fbea..cda7e35b0fa1c89b6d5b422193562d3ab777602c 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -25,7 +25,7 @@ bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) { } bool FORCE_INLINE walIsEmpty(SWal* pWal) { - return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) + return (pWal->vers.firstVer == -1 || pWal->vers.lastVer < pWal->vers.firstVer); // [firstVer, lastVer + 1) } int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } @@ -49,7 +49,6 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { int32_t sz = taosArrayGetSize(pWal->fileInfoSet); terrno = TSDB_CODE_SUCCESS; - ASSERT(fileIdx >= 0 && fileIdx < sz); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; @@ -101,7 +100,6 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { offsetBackward = offset; } - ASSERT(offset <= end); readSize = end - offset; capacity = readSize + sizeof(magic); @@ -257,7 +255,6 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) { SWalFileInfo* pLogInfo = taosArrayGet(actualLogList, i); while (j < metaFileNum) { SWalFileInfo* pMetaInfo = taosArrayGet(metaLogList, j); - ASSERT(pMetaInfo != NULL); if (pMetaInfo->firstVer < pLogInfo->firstVer) { j++; } else if (pMetaInfo->firstVer == pLogInfo->firstVer) { @@ -405,7 +402,6 @@ int walCheckAndRepairMeta(SWal* pWal) { taosArrayDestroy(actualLog); int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(sz == actualFileNum); // scan and determine the lastVer int32_t fileIdx = sz; @@ -423,8 +419,6 @@ int walCheckAndRepairMeta(SWal* pWal) { return -1; } - ASSERT(pFileInfo->firstVer >= 0); - if (pFileInfo->lastVer >= pFileInfo->firstVer && fileSize == pFileInfo->fileSize) { totSize += pFileInfo->fileSize; continue; @@ -439,7 +433,6 @@ int walCheckAndRepairMeta(SWal* pWal) { wError("failed to scan wal last ver since %s", terrstr()); return -1; } - ASSERT(pFileInfo->fileSize == 0); // remove the empty wal log, and its idx wInfo("vgId:%d, wal remove empty file %s", pWal->cfg.vgId, fnameStr); taosRemoveFile(fnameStr); @@ -499,8 +492,7 @@ int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { } int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { - int32_t sz = taosArrayGetSize(pWal->fileInfoSet); - ASSERT(fileIdx >= 0 && fileIdx < sz); + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); @@ -514,7 +506,6 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { return -1; } - ASSERT(pFileInfo->fileSize > 0 && pFileInfo->firstVer >= 0 && pFileInfo->lastVer >= pFileInfo->firstVer); if (fileSize == (pFileInfo->lastVer - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)) { return 0; } @@ -578,7 +569,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } offset += sizeof(SWalIdxEntry); - ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); + /*A(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry));*/ // ftruncate idx file if (offset < fileSize) { @@ -600,7 +591,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { int64_t count = 0; while (idxEntry.ver < pFileInfo->lastVer) { - ASSERT(idxEntry.ver == ckHead.head.version); + /*A(idxEntry.ver == ckHead.head.version);*/ idxEntry.ver += 1; idxEntry.offset += sizeof(SWalCkHead) + ckHead.head.bodyLen; @@ -677,8 +668,7 @@ int walRollFileInfo(SWal* pWal) { } char* walMetaSerialize(SWal* pWal) { - char buf[30]; - ASSERT(pWal->fileInfoSet); + char buf[30]; int sz = taosArrayGetSize(pWal->fileInfoSet); cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); @@ -735,7 +725,7 @@ char* walMetaSerialize(SWal* pWal) { } int walMetaDeserialize(SWal* pWal, const char* bytes) { - ASSERT(taosArrayGetSize(pWal->fileInfoSet) == 0); + /*A(taosArrayGetSize(pWal->fileInfoSet) == 0);*/ cJSON *pRoot, *pMeta, *pFiles, *pInfoJson, *pField; pRoot = cJSON_Parse(bytes); if (!pRoot) goto _err; @@ -853,7 +843,9 @@ int walSaveMeta(SWal* pWal) { // flush to a tmpfile n = walBuildTmpMetaName(pWal, tmpFnameStr); - ASSERT(n < sizeof(tmpFnameStr) && "Buffer overflow of file name"); + if (n >= sizeof(tmpFnameStr)) { + return -1; + } TdFilePtr pMetaFile = taosOpenFile(tmpFnameStr, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pMetaFile == NULL) { @@ -884,7 +876,9 @@ int walSaveMeta(SWal* pWal) { // rename it n = walBuildMetaName(pWal, metaVer + 1, fnameStr); - ASSERT(n < sizeof(fnameStr) && "Buffer overflow of file name"); + if (n >= sizeof(fnameStr)) { + goto _err; + } if (taosRenameFile(tmpFnameStr, fnameStr) < 0) { wError("failed to rename file due to %s. dest:%s", strerror(errno), fnameStr); @@ -907,7 +901,6 @@ _err: } int walLoadMeta(SWal* pWal) { - ASSERT(pWal->fileInfoSet->size == 0); // find existing meta file int metaVer = walFindCurMetaVer(pWal); if (metaVer == -1) { @@ -920,7 +913,7 @@ int walLoadMeta(SWal* pWal) { int64_t fileSize = 0; taosStatFile(fnameStr, &fileSize, NULL); if (fileSize == 0) { - taosRemoveFile(fnameStr); + (void)taosRemoveFile(fnameStr); wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer); return -1; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 526dba0bb52e891bfebaaf369a66b4109c3f49c8..7bfd3df9e0b7dba04d02226223bb88cd84fbda21 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -131,7 +131,6 @@ static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int return -1; } - ASSERT(entry.ver == ver); ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -268,55 +267,53 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { return 0; } -static int32_t walFetchBodyNew(SWalReader *pRead) { - SWalCont *pReadHead = &pRead->pHead->head; +static int32_t walFetchBodyNew(SWalReader *pReader) { + SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, index:%" PRId64, pRead->pWal->cfg.vgId, ver); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); - if (pRead->capacity < pReadHead->bodyLen) { - SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pRead->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); + if (pReader->capacity < pReadHead->bodyLen) { + SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pRead->pHead = ptr; - pReadHead = &pRead->pHead->head; - pRead->capacity = pReadHead->bodyLen; + pReader->pHead = ptr; + pReadHead = &pReader->pHead->head; + pReader->capacity = pReadHead->bodyLen; } - if (pReadHead->bodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, pReadHead->bodyLen)) { + if (pReadHead->bodyLen != taosReadFile(pReader->pLogFile, pReadHead->body, pReadHead->bodyLen)) { if (pReadHead->bodyLen < 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver, tstrerror(terrno)); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver, tstrerror(terrno)); } else { wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted", - pRead->pWal->cfg.vgId, pRead->pHead->head.version, ver); + pReader->pWal->cfg.vgId, pReader->pHead->head.version, ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } - pRead->curInvalid = 1; + pReader->curInvalid = 1; return -1; } - if (walValidBodyCksum(pRead->pHead) != 0) { - wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); - pRead->curInvalid = 1; + if (walValidBodyCksum(pReader->pHead) != 0) { + wError("vgId:%d, wal fetch body error:%" PRId64 ", since body checksum not passed", pReader->pWal->cfg.vgId, ver); + pReader->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pRead->pWal->cfg.vgId, ver); - pRead->curVersion = ver + 1; + wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); + pReader->curVersion = ver + 1; return 0; } static int32_t walSkipFetchBodyNew(SWalReader *pRead) { int64_t code; - ASSERT(pRead->curVersion == pRead->pHead->head.version); - ASSERT(pRead->curInvalid == 0); - code = taosLSeekFile(pRead->pLogFile, pRead->pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -394,9 +391,6 @@ int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { pRead->pWal->cfg.vgId, pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer); - ASSERT(pRead->curVersion == pHead->head.version); - ASSERT(pRead->curInvalid == 0); - code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/wal/src/walRef.c b/source/libs/wal/src/walRef.c index 768256cefa3dd0e8322eb163ca5107be5e48cab4..4d451db0c00bc5574573dc54ec4bb97462355a9c 100644 --- a/source/libs/wal/src/walRef.c +++ b/source/libs/wal/src/walRef.c @@ -26,7 +26,7 @@ SWalRef *walOpenRef(SWal *pWal) { } pRef->refId = tGenIdPI64(); pRef->refVer = -1; -// pRef->refFile = -1; + // pRef->refFile = -1; pRef->pWal = pWal; taosHashPut(pWal->pRefHash, &pRef->refId, sizeof(int64_t), &pRef, sizeof(void *)); return pRef; @@ -58,11 +58,11 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) { pRef->refVer = ver; // bsearch in fileSet -// SWalFileInfo tmpInfo; -// tmpInfo.firstVer = ver; -// SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); -// ASSERT(pRet != NULL); -// pRef->refFile = pRet->firstVer; + // SWalFileInfo tmpInfo; + // tmpInfo.firstVer = ver; + // SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + // ASSERT(pRet != NULL); + // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); } @@ -73,7 +73,7 @@ int32_t walRefVer(SWalRef *pRef, int64_t ver) { #if 1 void walUnrefVer(SWalRef *pRef) { pRef->refId = -1; -// pRef->refFile = -1; + // pRef->refFile = -1; } #endif @@ -88,11 +88,11 @@ SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { int64_t ver = walGetFirstVer(pWal); pRef->refVer = ver; // bsearch in fileSet -// SWalFileInfo tmpInfo; -// tmpInfo.firstVer = ver; -// SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); -// ASSERT(pRet != NULL); -// pRef->refFile = pRet->firstVer; + // SWalFileInfo tmpInfo; + // tmpInfo.firstVer = ver; + // SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + // ASSERT(pRet != NULL); + // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); wDebug("vgId:%d, wal ref version %" PRId64 " for first", pWal->cfg.vgId, ver); @@ -103,6 +103,7 @@ SWalRef *walRefFirstVer(SWal *pWal, SWalRef *pRef) { SWalRef *walRefCommittedVer(SWal *pWal) { SWalRef *pRef = walOpenRef(pWal); if (pRef == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } taosThreadMutexLock(&pWal->mutex); @@ -117,7 +118,7 @@ SWalRef *walRefCommittedVer(SWal *pWal) { tmpInfo.firstVer = ver; SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); -// pRef->refFile = pRet->firstVer; + // pRef->refFile = pRet->firstVer; taosThreadMutexUnlock(&pWal->mutex); return pRef; diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 2cb6614b01bab90cc41e0d11408f3380de4a95fc..cbfd0ef7417389b662ece46d4032f2f7a1a31cbd 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -40,7 +40,6 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - ASSERT(entry.ver == ver); code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -53,8 +52,7 @@ static int64_t walSeekWritePos(SWal* pWal, int64_t ver) { int walInitWriteFile(SWal* pWal) { TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); - ASSERT(pRet != NULL); - int64_t fileFirstVer = pRet->firstVer; + int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); @@ -109,9 +107,8 @@ int64_t walChangeWrite(SWal* pWal, int64_t ver) { tmpInfo.firstVer = ver; // bsearch in fileSet int32_t idx = taosArraySearchIdx(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); - ASSERT(idx != -1); + /*A(idx != -1);*/ SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, idx); - /*ASSERT(pFileInfo != NULL);*/ int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index b38961709e2ddf14b284c7dd3fa909724ad6a61c..a992c951fab0a5b9d2c0889d3fe4eb8588374ab3 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -63,7 +63,7 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { wInfo("vgId:%d, restore from snapshot, remove file %s", pWal->cfg.vgId, fnameStr); } } - walRemoveMeta(pWal); + (void)walRemoveMeta(pWal); pWal->writeCur = -1; pWal->totSize = 0; @@ -90,7 +90,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) { if (ver < pWal->vers.commitVer) { return 0; } - if (ver > pWal->vers.lastVer) { + if (ver > pWal->vers.lastVer || pWal->vers.commitVer < pWal->vers.snapshotVer) { terrno = TSDB_CODE_WAL_INVALID_VER; return -1; } @@ -121,7 +121,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // delete files in descending order int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); for (int i = pWal->writeCur + 1; i < fileSetSize; i++) { - SWalFileInfo* pInfo = taosArrayPop(pWal->fileInfoSet); + SWalFileInfo *pInfo = taosArrayPop(pWal->fileInfoSet); walBuildLogName(pWal, pInfo->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s for rollback", pWal->cfg.vgId, fnameStr); @@ -374,6 +374,7 @@ int32_t walEndSnapshot(SWal *pWal) { walBuildIdxName(pWal, pInfo->firstVer, fnameStr); wDebug("vgId:%d, wal remove file %s", pWal->cfg.vgId, fnameStr); if (taosRemoveFile(fnameStr) < 0 && errno != ENOENT) { + wError("vgId:%d, failed to remove idx file %s due to %s", pWal->cfg.vgId, fnameStr, strerror(errno)); goto END; } } @@ -445,7 +446,8 @@ END: static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); - int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); + + int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, idxOffset); diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index bc554c48a341f51ff4cf51306a7e5b45299f4282..db066a82b6647462eb3b17e7723411477f7fa1e1 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -43,7 +43,7 @@ target_link_libraries( ) if(TD_WINDOWS) target_link_libraries( - os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump dbghelp + os PUBLIC ws2_32 iconv msvcregex wcwidth winmm crashdump dbghelp version ) elseif(TD_DARWIN_64) find_library(CORE_FOUNDATION_FRAMEWORK CoreFoundation) @@ -53,6 +53,10 @@ elseif(TD_DARWIN_64) target_link_libraries( os PUBLIC dl m iconv ) +elseif(TD_ALPINE) + target_link_libraries( + os PUBLIC dl m rt unwind + ) else() target_link_libraries( os PUBLIC dl m rt diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 331d24174591648a080242b22b889866efa93eb4..3d63da7ba33adc05d201987faf438390b9bc6759 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -89,6 +89,8 @@ typedef struct dirent TdDirEntry; #endif +#define TDDIRMAXLEN 1024 + void taosRemoveDir(const char *dirname) { TdDirPtr pDir = taosOpenDir(dirname); if (pDir == NULL) return; @@ -133,8 +135,8 @@ int32_t taosMkDir(const char *dirname) { } int32_t taosMulMkDir(const char *dirname) { - if (dirname == NULL) return -1; - char temp[1024]; + if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) return -1; + char temp[TDDIRMAXLEN]; char *pos = temp; int32_t code = 0; #ifdef WINDOWS @@ -192,8 +194,8 @@ int32_t taosMulMkDir(const char *dirname) { } int32_t taosMulModeMkDir(const char *dirname, int mode) { - if (dirname == NULL) return -1; - char temp[1024]; + if (dirname == NULL || strlen(dirname) >= TDDIRMAXLEN) return -1; + char temp[TDDIRMAXLEN]; char *pos = temp; int32_t code = 0; #ifdef WINDOWS @@ -204,8 +206,7 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { #endif if (taosDirExist(temp)) { - chmod(temp, mode); - return code; + return chmod(temp, mode); } if (strncmp(temp, TD_DIRSEP, 1) == 0) { @@ -247,12 +248,10 @@ int32_t taosMulModeMkDir(const char *dirname, int mode) { } if (code < 0 && errno == EEXIST) { - chmod(temp, mode); - return 0; + return chmod(temp, mode); } - chmod(temp, mode); - return code; + return chmod(temp, mode); } void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 4cdf4cbdbc512fec2d774a442afe00a68f450bd2..aab547223f8d5caf11ec824eeeece978f934ae0d 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -132,15 +132,20 @@ int64_t taosCopyFile(const char *from, const char *to) { if (bytes < sizeof(buffer)) break; } - taosFsyncFile(pFileTo); + int code = taosFsyncFile(pFileTo); taosCloseFile(&pFileFrom); taosCloseFile(&pFileTo); + + if (code != 0) { + return -1; + } return size; _err: if (pFileFrom != NULL) taosCloseFile(&pFileFrom); if (pFileTo != NULL) taosCloseFile(&pFileTo); + /* coverity[+retval] */ taosRemoveFile(to); return -1; #endif @@ -151,7 +156,7 @@ TdFilePtr taosCreateFile(const char *path, int32_t tdFileOptions) { if (!fp) { if (errno == ENOENT) { // Try to create directory recursively - char *s = strdup(path); + char *s = taosStrdup(path); if (taosMulMkDir(taosDirName(s)) != 0) { taosMemoryFree(s); return NULL; @@ -192,7 +197,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { int32_t code = _stati64(path, &fileStat); #else struct stat fileStat; - int32_t code = stat(path, &fileStat); + int32_t code = stat(path, &fileStat); #endif if (code < 0) { return code; @@ -209,10 +214,9 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { return 0; } int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { - if (pFile == NULL) { - return 0; + if (pFile == NULL || pFile->fd < 0) { + return -1; } - assert(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS @@ -234,7 +238,7 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { #else struct stat fileStat; - int32_t code = fstat(pFile->fd, &fileStat); + int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { printf("taosFStatFile run fstat fail."); return code; @@ -266,7 +270,10 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } else { mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; } - assert(!(tdFileOptions & TD_FILE_EXCL)); + ASSERT(!(tdFileOptions & TD_FILE_EXCL)); + if (tdFileOptions & TD_FILE_EXCL) { + return NULL; + } fp = fopen(path, mode); if (fp == NULL) { return NULL; @@ -365,7 +372,13 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -409,7 +422,13 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return -1; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -432,6 +451,9 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { taosThreadRwlockWrlock(&(pFile->rwlock)); #endif if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif return 0; } @@ -467,7 +489,13 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { +#if FILE_WITH_LOCK + taosThreadRwlockUnlock(&(pFile->rwlock)); +#endif + return 0; + } #ifdef WINDOWS size_t pos = _lseeki64(pFile->fd, 0, SEEK_CUR); _lseeki64(pFile->fd, offset, SEEK_SET); @@ -483,10 +511,13 @@ int64_t taosPWriteFile(TdFilePtr pFile, const void *buf, int64_t count, int64_t } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL || pFile->fd < 0) { + return -1; + } #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. #ifdef WINDOWS int64_t ret = _lseeki64(pFile->fd, offset, whence); #else @@ -502,7 +533,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { if (pFile == NULL) { return 0; } - assert(pFile->fd >= 0); // Please check if you have closed the file. + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } struct stat fileStat; #ifdef WINDOWS @@ -526,6 +560,10 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); // Please check if you have closed the file. + if (pFile->fd < 0) { + return -1; + } #ifdef WINDOWS BOOL fSuccess = FALSE; LARGE_INTEGER fileSize; @@ -544,17 +582,19 @@ int32_t taosLockFile(TdFilePtr pFile) { } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif } int32_t taosUnLockFile(TdFilePtr pFile) { + ASSERT(pFile->fd >= 0); + if (pFile->fd < 0) { + return 0; + } #ifdef WINDOWS - BOOL fSuccess = FALSE; - OVERLAPPED overlapped = {0}; - HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); + BOOL fSuccess = FALSE; + OVERLAPPED overlapped = {0}; + HANDLE hFile = (HANDLE)_get_osfhandle(pFile->fd); fSuccess = UnlockFileEx(hFile, 0, ~0, ~0, &overlapped); if (!fSuccess) { @@ -562,19 +602,19 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } return 0; #else - assert(pFile->fd >= 0); // Please check if you have closed the file. - return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#ifdef WINDOWS + if (pFile == NULL) { + return 0; + } if (pFile->fd < 0) { - errno = EBADF; printf("Ftruncate file error, fd arg was negative\n"); return -1; } +#ifdef WINDOWS HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); @@ -619,11 +659,6 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { return 0; #else - if (pFile == NULL) { - return 0; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - return ftruncate(pFile->fd, l_size); #endif } @@ -651,7 +686,10 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in if (pFileOut == NULL || pFileIn == NULL) { return 0; } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); + ASSERT(pFileIn->fd >= 0 && pFileOut->fd >= 0); + if (pFileIn->fd < 0 || pFileOut->fd < 0) { + return 0; + } #ifdef WINDOWS @@ -744,11 +782,9 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { - if (pFile == NULL) { + if (pFile == NULL || pFile->fp == NULL) { return; } - assert(pFile->fp != NULL); - va_list ap; va_start(ap, format); vfprintf(pFile->fp, format, ap); @@ -773,7 +809,10 @@ int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } #ifdef WINDOWS *ptrBuf = taosMemoryMalloc(1024); if (*ptrBuf == NULL) return -1; @@ -793,7 +832,10 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { if (pFile == NULL || buf == NULL) { return -1; } - assert(pFile->fp != NULL); + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; + } if (fgets(buf, maxSize, pFile->fp) == NULL) { return -1; } @@ -802,9 +844,12 @@ int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf) { int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { - return 0; + return -1; + } + ASSERT(pFile->fp != NULL); + if (pFile->fp == NULL) { + return -1; } - assert(pFile->fp != NULL); return feof(pFile->fp); } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 7319181a777cb8140396f592009507a151d2620b..7008c385763c61fd3343b4afd42db08623ebee40 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -59,11 +59,11 @@ char *taosCharsetReplace(char *charsetstr) { for (int32_t i = 0; i < tListLen(charsetRep); ++i) { if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); + return taosStrdup(charsetRep[i].newCharset); } } - return strdup(charsetstr); + return taosStrdup(charsetstr); } /** @@ -71,7 +71,7 @@ char *taosCharsetReplace(char *charsetstr) { * seems does not response as expected. * * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. + * both the Server and the Client may be interrupted. * * In case that the setLocale failed to be executed, the right charset needs to be set. */ diff --git a/source/os/src/osMath.c b/source/os/src/osMath.c index 41c8c9257a95fb57213c33042ac7c1339295553b..b466f89a1d127db23a1618f3455d876f25643f97 100644 --- a/source/os/src/osMath.c +++ b/source/os/src/osMath.c @@ -15,8 +15,8 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE -#include "os.h" #include +#include "talgo.h" #ifdef WINDOWS void swapStr(char* j, char* J, int width) { @@ -32,7 +32,7 @@ void swapStr(char* j, char* J, int width) { } #endif -int qsortHelper(const void* p1, const void* p2, const void* param) { +int32_t qsortHelper(const void* p1, const void* p2, const void* param) { __compar_fn_t comparFn = param; return comparFn(p1, p2); } @@ -41,9 +41,8 @@ int qsortHelper(const void* p1, const void* p2, const void* param) { void taosSort(void* base, int64_t sz, int64_t width, __compar_fn_t compar) { #ifdef _ALPINE void* param = compar; - taosqsort(base, width, sz, param, qsortHelper); + taosqsort(base, sz, width, param, qsortHelper); #else qsort(base, sz, width, compar); #endif } - diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 2e68bd5ccd3ad937dbcfce6988fcb7ef6060f31a..1ae4afe0e033731d26072a02e51a4fe34dc0a30f 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -228,6 +228,32 @@ void taosPrintBackTrace() { void taosPrintBackTrace() { return; } #endif +int32_t taosMemoryDbgInit() { +#if defined(LINUX) && !defined(_ALPINE) + int ret = mallopt(M_MMAP_THRESHOLD, 0); + if (0 == ret) { + return TAOS_SYSTEM_ERROR(errno); + } + + return 0; +#else + return TSDB_CODE_FAILED; +#endif +} + +int32_t taosMemoryDbgInitRestore() { +#if defined(LINUX) && !defined(_ALPINE) + int ret = mallopt(M_MMAP_THRESHOLD, 128 * 1024); + if (0 == ret) { + return TAOS_SYSTEM_ERROR(errno); + } + + return 0; +#else + return TSDB_CODE_FAILED; +#endif +} + void *taosMemoryMalloc(int64_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); @@ -266,7 +292,10 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { if (ptr == NULL) return taosMemoryMalloc(size); TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (tpTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } TdMemoryInfo tdMemoryInfo; memcpy(&tdMemoryInfo, pTdMemoryInfo, sizeof(TdMemoryInfo)); @@ -283,13 +312,15 @@ void *taosMemoryRealloc(void *ptr, int64_t size) { #endif } -void *taosMemoryStrDup(const char *ptr) { +char *taosStrdup(const char *ptr) { #ifdef USE_TD_MEMORY if (ptr == NULL) return NULL; TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); - + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { + return NULL; + } void *tmp = tstrdup(pTdMemoryInfo); if (tmp == NULL) return NULL; @@ -323,7 +354,10 @@ int64_t taosMemorySize(void *ptr) { #ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char *)ptr - sizeof(TdMemoryInfo)); - assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + ASSERT(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); + if (pTdMemoryInfo->symbol != TD_MEMORY_SYMBOL) { ++ return NULL; ++ } return pTdMemoryInfo->memorySize; #else @@ -338,7 +372,7 @@ int64_t taosMemorySize(void *ptr) { } void taosMemoryTrim(int32_t size) { -#if defined(WINDOWS) || defined(DARWIN) +#if defined(WINDOWS) || defined(DARWIN) || defined(_ALPINE) // do nothing return; #else @@ -348,7 +382,7 @@ void taosMemoryTrim(int32_t size) { void* taosMemoryMallocAlign(uint32_t alignment, int64_t size) { #ifdef USE_TD_MEMORY - assert(0); + ASSERT(0); #else #if defined(LINUX) void* p = memalign(alignment, size); diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1f2df09ce109d7aced2e14d9e8affe8fd3229836..c1ef57e9c5cc874ccc28c8228836693cb7de0db1 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -86,8 +86,8 @@ int32_t tsem_timewait(tsem_t* sem, int64_t ms) { while ((rc = sem_timedwait(sem, &ts)) == -1 && errno == EINTR) continue; return rc; /* This should have timed out */ - // assert(errno == ETIMEDOUT); - // assert(rc != 0); + // ASSERT(errno == ETIMEDOUT); + // ASSERT(rc != 0); // GetSystemTimeAsFileTime(&ft_after); // // We specified a non-zero wait. Time must advance. // if (ft_before.dwLowDateTime == ft_after.dwLowDateTime && ft_before.dwHighDateTime == ft_after.dwHighDateTime) diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 3a93a26b8576976a96d6a523575413ee854fa683..7d2c8aa4e5454c4c9cf22fe0f7bdb40ab90690ac 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -298,7 +298,7 @@ int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return -1; } #ifdef WINDOWS - assert(0); + ASSERT(0); return 0; #else return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); @@ -662,7 +662,7 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { int taosGetLocalIp(const char *eth, char *ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int fd; @@ -689,7 +689,7 @@ int taosGetLocalIp(const char *eth, char *ip) { int taosValidIp(uint32_t ip) { #if defined(WINDOWS) // DO NOTHAING - assert(0); + ASSERT(0); return 0; #else int ret = -1; @@ -745,8 +745,10 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { #endif serverAdd.sin_port = (uint16_t)htons(port); - if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { - // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (fd < 0) { // exception + return false; + } else if (fd <= 2) { // in, out, err taosCloseSocketNoCheck1(fd); return false; } @@ -924,7 +926,7 @@ uint32_t ip2uint(const char *const ip_addr) { void taosBlockSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); @@ -994,7 +996,7 @@ int32_t taosGetFqdn(char *fqdn) { #else printf("failed to get hostname, reason:%s\n", strerror(errno)); #endif - assert(0); + ASSERT(0); return -1; } @@ -1031,7 +1033,7 @@ void taosIgnSIGPIPE() { signal(SIGPIPE, SIG_IGN); } void taosSetMaskSIGPIPE() { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #else sigset_t signal_mask; sigemptyset(&signal_mask); diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5419da1c0ce49fe8e61cbf030ad540918835f465..ae4a8a5cada3789c05e09a7b59455b9a38f54df7 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -112,7 +112,7 @@ int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes) { } TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) { - assert(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); + ASSERT(taosMemorySize(target_ucs4) >= len_ucs4 * sizeof(TdUcs4)); return memcpy(target_ucs4, source_ucs4, len_ucs4 * sizeof(TdUcs4)); } @@ -217,7 +217,7 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else memset(ucs4, 0, ucs4_max_len); @@ -245,7 +245,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4 int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return -1; #else @@ -263,7 +263,7 @@ int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs) { } bool taosValidateEncodec(const char *encodec) { #ifdef DISALLOW_NCHAR_WITHOUT_ICONV - printf("Nchar cannot be read and written without iconv, please install iconv library and recompile TDengine.\n"); + printf("Nchar cannot be read and written without iconv, please install iconv library and recompile.\n"); return true; #else iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); @@ -351,75 +351,75 @@ char *taosStrCaseStr(const char *str, const char *pattern) { int64_t taosStr2Int64(const char *str, char **pEnd, int32_t radix) { int64_t tmp = strtoll(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } uint64_t taosStr2UInt64(const char *str, char **pEnd, int32_t radix) { uint64_t tmp = strtoull(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } int32_t taosStr2Int32(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } uint32_t taosStr2UInt32(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtol(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); #endif return tmp; } int16_t taosStr2Int16(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SHRT_MIN); - assert(tmp <= SHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SHRT_MIN); + ASSERT(tmp <= SHRT_MAX); #endif return (int16_t)tmp; } uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= USHRT_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= USHRT_MAX); #endif return (uint16_t)tmp; } @@ -427,23 +427,23 @@ uint16_t taosStr2UInt16(const char *str, char **pEnd, int32_t radix) { int8_t taosStr2Int8(const char *str, char **pEnd, int32_t radix) { int32_t tmp = strtol(str, pEnd, radix); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp >= SCHAR_MIN); - assert(tmp <= SCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp >= SCHAR_MIN); + ASSERT(tmp <= SCHAR_MAX); #endif return tmp; } uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { uint32_t tmp = strtoul(str, pEnd, radix); -#ifdef DARWIN +#if defined(DARWIN) || defined(_ALPINE) if (errno == EINVAL) errno = 0; #endif #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp <= UCHAR_MAX); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp <= UCHAR_MAX); #endif return tmp; } @@ -451,9 +451,9 @@ uint8_t taosStr2UInt8(const char *str, char **pEnd, int32_t radix) { double taosStr2Double(const char *str, char **pEnd) { double tmp = strtod(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VAL); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VAL); #endif return tmp; } @@ -461,10 +461,10 @@ double taosStr2Double(const char *str, char **pEnd) { float taosStr2Float(const char *str, char **pEnd) { float tmp = strtof(str, pEnd); #ifdef TD_CHECK_STR_TO_INT_ERROR - assert(errno != ERANGE); - assert(errno != EINVAL); - assert(tmp != HUGE_VALF); - assert(tmp != NAN); + ASSERT(errno != ERANGE); + ASSERT(errno != EINVAL); + ASSERT(tmp != HUGE_VALF); + ASSERT(tmp != NAN); #endif return tmp; } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 897e68a12687e3ae713f685a2898a4061bb78f81..52309a7b35cbdad87baf87ab10fae2bf42c1e335 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -124,7 +124,7 @@ static char tsProcIOFile[25] = {0}; static void taosGetProcIOnfos() { tsPageSizeKB = sysconf(_SC_PAGESIZE) / 1024; tsOpenMax = sysconf(_SC_OPEN_MAX); - tsStreamMax = sysconf(_SC_STREAM_MAX); + tsStreamMax = TMAX(sysconf(_SC_STREAM_MAX), 0); tsProcId = (pid_t)syscall(SYS_gettid); snprintf(tsProcMemFile, sizeof(tsProcMemFile), "/proc/%d/status", tsProcId); @@ -250,7 +250,7 @@ void taosGetSystemInfo() { int32_t taosGetEmail(char *email, int32_t maxLen) { #ifdef WINDOWS - // assert(0); + // ASSERT(0); #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -280,11 +280,46 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { #endif } +#ifdef WINDOWS +bool getWinVersionReleaseName(char *releaseName, int32_t maxLen) { + TCHAR szFileName[MAX_PATH]; + DWORD dwHandle; + DWORD dwLen; + LPVOID lpData; + UINT uLen; + VS_FIXEDFILEINFO *pFileInfo; + + GetWindowsDirectory(szFileName, MAX_PATH); + wsprintf(szFileName, L"%s%s", szFileName, L"\\explorer.exe"); + dwLen = GetFileVersionInfoSize(szFileName, &dwHandle); + if (dwLen == 0) { + return false; + } + lpData = malloc(dwLen); + if (lpData == NULL) return false; + if (!GetFileVersionInfo(szFileName, dwHandle, dwLen, lpData)) { + free(lpData); + return false; + } + + if (!VerQueryValue(lpData, L"\\", (LPVOID *)&pFileInfo, &uLen)) { + free(lpData); + return false; + } + + snprintf(releaseName, maxLen, "Windows %d.%d", HIWORD(pFileInfo->dwProductVersionMS), + LOWORD(pFileInfo->dwProductVersionMS)); + free(lpData); + return true; +} +#endif int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { #ifdef WINDOWS - snprintf(releaseName, maxLen, "Windows"); + if (!getWinVersionReleaseName(releaseName, maxLen)) { + snprintf(releaseName, maxLen, "Windows"); + } return 0; #elif defined(_TD_DARWIN_64) char osversion[32]; @@ -358,6 +393,10 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { code = 0; done |= 1; } + int endPos = strlen(cpuModel)-1; + if (cpuModel[endPos] == '\n') { + cpuModel[endPos] = '\0'; + } taosCloseCmd(&pCmd); pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count"); @@ -400,11 +439,14 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { if (code != 0 && (done & 1) == 0) { TdFilePtr pFile1 = taosOpenFile("/proc/device-tree/model", TD_FILE_READ | TD_FILE_STREAM); - if (pFile1 == NULL) return code; - taosGetsFile(pFile1, maxLen, cpuModel); - taosCloseFile(&pFile1); - code = 0; - done |= 1; + if (pFile1 != NULL) { + ssize_t bytes = taosGetsFile(pFile1, maxLen, cpuModel); + taosCloseFile(&pFile); + if (bytes > 0) { + code = 0; + done |= 1; + } + } } if (code != 0 && (done & 1) == 0) { @@ -459,7 +501,7 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { curSysTotal = curSysUsed + sysCpu.idle; curProcTotal = procCpu.utime + procCpu.stime + procCpu.cutime + procCpu.cstime; - if (curSysTotal > lastSysTotal && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { + if (curSysTotal - lastSysTotal > 0 && curSysUsed >= lastSysUsed && curProcTotal >= lastProcTotal) { if (cpu_system != NULL) { *cpu_system = (curSysUsed - lastSysUsed) / (double)(curSysTotal - lastSysTotal) * 100; } @@ -571,12 +613,6 @@ int32_t taosGetProcMemory(int64_t *usedKB) { } } - if (strlen(line) < 0) { - // printf("read file:%s failed", tsProcMemFile); - taosCloseFile(&pFile); - return -1; - } - char tmp[10]; sscanf(line, "%s %" PRId64, tmp, usedKB); @@ -865,7 +901,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { char *taosGetCmdlineByPID(int pid) { #ifdef WINDOWS - assert(0); + ASSERT(0); return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index c972aebbcaf8e89b7104e9c9241b5f7d11b6449d..7b6c77f7bc7ec969438689068a521ed20c123940 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -90,7 +90,7 @@ typedef struct FILE TdCmd; void* taosLoadDll(const char* filename) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -109,7 +109,7 @@ void* taosLoadDll(const char* filename) { void* taosLoadSym(void* handle, char* name) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return NULL; #elif defined(_TD_DARWIN_64) return NULL; @@ -130,7 +130,7 @@ void* taosLoadSym(void* handle, char* name) { void taosCloseDll(void* handle) { #if defined(WINDOWS) - assert(0); + ASSERT(0); return; #elif defined(_TD_DARWIN_64) return; diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 32c58695cfdac108e38112ccbfc73f35faf5ed42..39ba92fdc5e93d45facd85a2015461ad1fd68d8c 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -233,7 +233,8 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { #ifdef TD_USE_SPINLOCK_AS_MUTEX - assert(pshared == 0); + ASSERT(pshared == 0); + if (pshared != 0) return -1; return pthread_mutex_init((pthread_mutex_t *)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t *)lock, pshared); diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index d1c233ea9c2a06de6f4f973395276fcae6246188..5a57007d1765302994c091bc3f970ef98388e000 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -99,8 +99,8 @@ static void *taosProcessAlarmSignal(void *tharg) { setThreadName("tmr"); #ifdef _ALPINE - sevent.sigev_notify = SIGEV_THREAD; - sevent.sigev_value.sival_int = syscall(__NR_gettid); + sevent.sigev_notify = SIGEV_THREAD_ID; + sevent.sigev_notify_thread_id = syscall(__NR_gettid); #else sevent.sigev_notify = SIGEV_THREAD_ID; sevent._sigev_un._tid = syscall(__NR_gettid); diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index ab5600744c887d2fb39bc2bbb3bd30ba007fecf7..ad223bff27de80a454d09215926dcdf44e95c23c 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -909,7 +909,7 @@ void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { char buf[4096] = {0}; char *tz = NULL; { - int n = readlink("/etc/localtime", buf, sizeof(buf)); + int n = readlink("/etc/localtime", buf, sizeof(buf)-1); if (n < 0) { printf("read /etc/localtime error, reason:%s", strerror(errno)); diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 2e24bb05269ba9a4529b16726ca8c59d6de7ac68..1d6542e78cd987e5228c94609cf0dd865a6d6396 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -34,6 +34,12 @@ TEST(osTest, osSystem) { ELogLevel level = DEBUG_FATAL; int32_t dflag = 255; // tsLogEmbedded ? 255 : uDebugFlag taosPrintTrace(flags, level, dflag, 0); + + const int sysLen = 64; + char osSysName[sysLen]; + int ret = taosGetOsReleaseName(osSysName, sysLen); + printf("os systeme name:%s\n", osSysName); + ASSERT_EQ(ret, 0); } void fileOperateOnFree(void *param) { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 07b3084222cfd8f38896e53e4ab1cd481efe2ff4..7467fa29481d932632b612fc44550fd024542fc3 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -212,6 +212,8 @@ void* taosArrayReserve(SArray* pArray, int32_t num) { void* dst = TARRAY_GET_ELEM(pArray, pArray->size); pArray->size += num; + memset(dst, 0, num * pArray->elemSize); + return dst; } @@ -257,7 +259,7 @@ size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { return 0; } - return pArray->size; + return TARRAY_SIZE(pArray); } void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { @@ -321,6 +323,20 @@ void taosArrayRemove(SArray* pArray, size_t index) { pArray->size -= 1; } +void taosArrayRemoveBatch(SArray* pArray, size_t index, size_t num, FDelete fp) { + ASSERT(index + num <= pArray->size); + + if (fp) { + for (int32_t i = 0; i < num; i++) { + fp(taosArrayGet(pArray, index + i)); + } + } + + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + num) * pArray->elemSize, + (pArray->size - index - num) * pArray->elemSize); + pArray->size -= num; +} + SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { if (elemSize <= 0) { terrno = TSDB_CODE_INVALID_PARA; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 761da6986ba635cd1bafa2b49d8719fb930b1992..1d480e7beb0a810f25cc969009bc0901e32719b5 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -391,7 +391,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLi return NULL; } - pCacheObj->name = strdup(cacheName); + pCacheObj->name = taosStrdup(cacheName); doRegisterCacheObj(pCacheObj); return pCacheObj; } diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 46a905143697e08021bf400cf3e49b82f05e5e83..f8f78ae6a5b3df99409694c67cc322cffa762cef 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -17,11 +17,11 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE #include "tcompare.h" -#include "tutil.h" #include "regex.h" #include "tdef.h" #include "thash.h" #include "tlog.h" +#include "tutil.h" #include "types.h" #include "osString.h" @@ -1010,7 +1010,8 @@ int32_t compareJsonValDesc(const void *pLeft, const void *pRight) { return compa * '_': Matches one character * */ -int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo) { +int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, + const SPatternCompareInfo *pInfo) { char c, c1; int32_t i = 0; @@ -1020,9 +1021,9 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t while ((i < psize) && ((c = pattern[i++]) != 0)) { if (c == pInfo->matchAll) { /* Match "*" */ - while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) { + while ((i < psize) && ((c = pattern[i++]) == pInfo->matchAll || c == pInfo->matchOne)) { if (c == pInfo->matchOne) { - if (j >= ssize || str[j++] == 0) { // empty string, return not match + if (j >= ssize || str[j++] == 0) { // empty string, return not match return TSDB_PATTERN_NOWILDCARDMATCH; } else { ++nMatchChar; @@ -1077,7 +1078,8 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t return (j >= ssize || str[j] == 0) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo) { +int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, + const SPatternCompareInfo *pInfo) { TdUcs4 c, c1; int32_t i = 0; @@ -1189,10 +1191,11 @@ int32_t comparestrRegexMatch(const void *pLeft, const void *pRight) { taosMemoryFree(str); taosMemoryFree(pattern); - return (ret == 0) ? 0 : 1;; + return (ret == 0) ? 0 : 1; + ; } -int32_t comparewcsRegexMatch(const void* pString, const void* pPattern) { +int32_t comparewcsRegexMatch(const void *pString, const void *pPattern) { size_t len = varDataLen(pPattern); char *pattern = taosMemoryMalloc(len + 1); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 609a386367d825bca096b12857b8e8ece013c49d..c15bd96903c901ea9f0c7dd6aae317926c2b8ac9 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -109,7 +109,7 @@ int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); } static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { cfgFreeItem(pItem); - pItem->str = strdup(timezone); + pItem->str = taosStrdup(timezone); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -120,7 +120,7 @@ static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { cfgFreeItem(pItem); - pItem->str = strdup(charset); + pItem->str = taosStrdup(charset); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -131,7 +131,7 @@ static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { cfgFreeItem(pItem); - pItem->str = strdup(locale); + pItem->str = taosStrdup(locale); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -149,7 +149,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { } taosMemoryFreeClear(pItem->str); - pItem->str = strdup(fullDir); + pItem->str = taosStrdup(fullDir); if (pItem->str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -215,7 +215,7 @@ static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType st } static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - char *tmp = strdup(value); + char *tmp = taosStrdup(value); if (tmp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s", pItem->name, cfgDtypeStr(pItem->dtype), @@ -358,7 +358,7 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; - pItem->name = strdup(name); + pItem->name = taosStrdup(name); if (pItem->name == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -417,7 +417,7 @@ int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double mi int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; - item.str = strdup(defaultVal); + item.str = taosStrdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b85035ffcfa1728b4845d18455085349fd8b2c98..4bb082800dd52da9c325df7f91ed6ad52baf9cc5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -97,6 +97,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TIMEOUT_ERROR, "Operation timeout") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") +TAOS_DEFINE_ERROR(TSDB_CODE_IVLD_DATA_FMT, "Invalid data format") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -139,6 +140,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt cl TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node in current query policy configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") @@ -207,6 +210,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "Tag index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "Tag index not exists") + + // mnode-db TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") @@ -290,6 +297,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_MULTI_REPLICA_SOURCE_DB, "Stream temporarily does not support source db having replica > 1") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STREAMS, "Too many streams") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TARGET_TABLE, "Cannot write the same stable as other stream") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") @@ -321,6 +329,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_SUBSCRIBED, "Table column is subsc TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_AVAIL_BUFPOOL, "No availabe buffer pool") TAOS_DEFINE_ERROR(TSDB_CODE_VND_STOPPED, "Vnode stopped") TAOS_DEFINE_ERROR(TSDB_CODE_VND_DUP_REQUEST, "Duplicate write request") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_QUERY_BUSY, "Query busy") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") @@ -580,6 +589,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp p TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data format") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") TAOS_DEFINE_ERROR(TSDB_CODE_SML_NOT_SAME_TYPE, "Not the same type like before") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INTERNAL_ERROR, "Internal error") //tsma TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_INIT_FAILED, "Tsma init failed") @@ -595,6 +605,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSMA_NO_INDEX_IN_CACHE, "No tsma index in ca TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_ENV, "Invalid rsma env") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_STAT, "Invalid rsma state") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_QTASKINFO_CREATE, "Rsma qtaskinfo creation error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_COMMIT, "Rsma fs commit error") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") @@ -602,6 +613,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_OPEN, "Rsma stream state open") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_STREAM_STATE_COMMIT, "Rsma stream state commit") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_REF, "Rsma fs ref error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_SYNC, "Rsma fs sync error") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FS_UPDATE, "Rsma fs update error") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 33182eca49bba0b3a10fe0ad409f6737f0800c3e..0b7ad330a7f5c084618650bc6c5f99ec96ea6b88 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -192,7 +192,7 @@ int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) if (NULL == p) { return TSDB_CODE_SUCCESS; } - *pVal = strdup(p); + *pVal = taosStrdup(p); return TSDB_CODE_SUCCESS; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 6bcf4ad39b729786e2d562d7a928b0a4965dc03d..fa8b5d33b7e8f45fe37d658f1ba336d21a43e5b4 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -55,7 +55,7 @@ static int32_t createDiskFile(SDiskbasedBuf* pBuf) { if (pBuf->path == NULL) { // prepare the file name when needed it char path[PATH_MAX] = {0}; taosGetTmpfilePath(pBuf->prefix, "paged-buf", path); - pBuf->path = taosMemoryStrDup(path); + pBuf->path = taosStrdup(path); if (pBuf->path == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -351,7 +351,7 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem pPBuf->totalBufSize = 0; pPBuf->allocateId = -1; pPBuf->pFile = NULL; - pPBuf->id = strdup(id); + pPBuf->id = taosStrdup(id); pPBuf->fileSize = 0; pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); pPBuf->freePgList = tdListNew(POINTER_BYTES); @@ -391,7 +391,7 @@ _error: return TSDB_CODE_OUT_OF_MEMORY; } -static char* doExtractPage(SDiskbasedBuf* pBuf) { +static char* doExtractPage(SDiskbasedBuf* pBuf, bool* newPage) { char* availablePage = NULL; if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { availablePage = evictBufPage(pBuf); @@ -405,6 +405,7 @@ static char* doExtractPage(SDiskbasedBuf* pBuf) { if (availablePage == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; } + *newPage = true; } return availablePage; @@ -413,7 +414,8 @@ static char* doExtractPage(SDiskbasedBuf* pBuf) { void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { pBuf->statis.getPages += 1; - char* availablePage = doExtractPage(pBuf); + bool newPage = false; + char* availablePage = doExtractPage(pBuf, &newPage); if (availablePage == NULL) { return NULL; } @@ -432,6 +434,9 @@ void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t* pageId) { // register page id info pi = registerNewPageInfo(pBuf, *pageId); if (pi == NULL) { + if (newPage) { + taosMemoryFree(availablePage); + } return NULL; } @@ -492,7 +497,8 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { ASSERT((!BUF_PAGE_IN_MEM(*pi)) && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1))); - (*pi)->pData = doExtractPage(pBuf); + bool newPage = false; + (*pi)->pData = doExtractPage(pBuf, &newPage); // failed to evict buffer page, return with error code. if ((*pi)->pData == NULL) { @@ -509,6 +515,10 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { if (HAS_DATA_IN_DISK(*pi)) { int32_t code = loadPageFromDisk(pBuf, *pi); if (code != 0) { + if (newPage) { + taosMemoryFree((*pi)->pData); + } + terrno = code; return NULL; } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 42b63588930712d2e132fd7e8bf860aef3622ff9..3769da6ccd60a2fe4a6a949ae9b66e8a5a3e1486 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -339,7 +339,6 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { STaosQueue *prev = qset->head; tqueue = qset->head->next; while (tqueue) { - assert(tqueue->qset); if (tqueue == queue) { prev->next = tqueue->next; break; diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index ffae5441aa088f2cc71710b301d5243cb3975235..e7386d5912dd83c5a76af3c902bcd910b5ffef87 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -147,6 +147,10 @@ static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) { pTree->cmprFn = cmprFn; + tRBTreeClear(pTree); +} + +void tRBTreeClear(SRBTree *pTree) { pTree->n = 0; pTree->NIL = &pTree->NILNODE; pTree->NIL->color = BLACK; @@ -423,6 +427,22 @@ SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { return pNode; } +SRBTreeNode *tRBTreeDropMin(SRBTree *pTree) { + SRBTreeNode *pNode = tRBTreeMin(pTree); + if (pNode) { + tRBTreeDrop(pTree, pNode); + } + return pNode; +} + +SRBTreeNode *tRBTreeDropMax(SRBTree *pTree) { + SRBTreeNode *pNode = tRBTreeMax(pTree); + if (pNode) { + tRBTreeDrop(pTree, pNode); + } + return pNode; +} + SRBTreeNode *tRBTreeGet(SRBTree *pTree, const SRBTreeNode *pKeyNode) { SRBTreeNode *pNode = pTree->root; diff --git a/tests/develop-test/2-query/table_count_scan.py b/tests/develop-test/2-query/table_count_scan.py index 5bdc915cdde07e513c9a65fbbcd7cd35e2a93a42..758d28948dd38ee6d411e08412cdc4a8d7906a39 100644 --- a/tests/develop-test/2-query/table_count_scan.py +++ b/tests/develop-test/2-query/table_count_scan.py @@ -14,7 +14,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVer=1): tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) self._conn = conn def restartTaosd(self, index=1, dbname="db"): @@ -65,7 +65,7 @@ class TDTestCase: tdSql.query('select count(*),db_name, stable_name from information_schema.ins_tables group by db_name, stable_name;') tdSql.checkRows(3) - tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 0, 24) tdSql.checkData(0, 1, 'information_schema') tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 3) @@ -77,7 +77,7 @@ class TDTestCase: tdSql.query('select count(1) v,db_name, stable_name from information_schema.ins_tables group by db_name, stable_name order by v desc;') tdSql.checkRows(3) - tdSql.checkData(0, 0, 23) + tdSql.checkData(0, 0, 24) tdSql.checkData(0, 1, 'information_schema') tdSql.checkData(0, 2, None) tdSql.checkData(1, 0, 5) @@ -93,7 +93,7 @@ class TDTestCase: tdSql.checkData(1, 1, 'performance_schema') tdSql.checkData(0, 0, 3) tdSql.checkData(0, 1, 'tbl_count') - tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 0, 24) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -106,7 +106,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 31) + tdSql.checkData(0, 0, 32) tdSql.execute('create table stba (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 float, c7 double, c8 binary(10), c9 nchar(10), c10 tinyint unsigned, c11 smallint unsigned, c12 int unsigned, c13 bigint unsigned) TAGS(t1 int, t2 binary(10), t3 double);') @@ -189,7 +189,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 23) + tdSql.checkData(3, 0, 24) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -204,7 +204,7 @@ class TDTestCase: tdSql.checkData(2, 0, 5) tdSql.checkData(2, 1, 'performance_schema') tdSql.checkData(2, 2, None) - tdSql.checkData(3, 0, 23) + tdSql.checkData(3, 0, 24) tdSql.checkData(3, 1, 'information_schema') tdSql.checkData(3, 2, None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(0, 1, 'tbl_count') tdSql.checkData(1, 0, 5) tdSql.checkData(1, 1, 'performance_schema') - tdSql.checkData(2, 0, 23) + tdSql.checkData(2, 0, 24) tdSql.checkData(2, 1, 'information_schema') tdSql.query("select count(*) from information_schema.ins_tables where db_name='tbl_count'") @@ -228,7 +228,7 @@ class TDTestCase: tdSql.query('select count(*) from information_schema.ins_tables') tdSql.checkRows(1) - tdSql.checkData(0, 0, 32) + tdSql.checkData(0, 0, 33) tdSql.execute('drop database tbl_count') diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py index 080d6aea8731562c1aaa4a6aacb6aca726059b1a..54657995e2b0abcfa80b1b08be0b40b9bf415126 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/insert_alltypes_json.py @@ -320,7 +320,7 @@ class TDTestCase: tdSql.checkData(0, 0, 160) tdSql.query("select count(*) from db.stb where t13 like 'b1%' or t13 like 'b2%'") tdSql.checkData(0, 0, 160) - + tAdapter.stop() def stop(self): diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_json_alltypes.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_json_alltypes.json index 5a373888a6cdfe2ee8a4880863459e91c87a8fa2..26a08b3fee7af72361229b944de7fefe7f8a1317 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_json_alltypes.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_json_alltypes.json @@ -16,13 +16,7 @@ "databases": [{ "dbinfo": { "name": "db", - "drop": "yes", - "replica": 1, - "precision": "ms", - "keep": 36500, - "minRows": 100, - "maxRows": 4096, - "comp":2 + "drop": "yes" }, "super_tables": [{ "name": "stb1", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py index c4b1750340eb73016e0fa623bdc484a0d33788a4..c906f6167526c707f1a065b32dfd65fe5bdc7eac 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/query_json.py @@ -120,7 +120,7 @@ class TDTestCase: def stop(self): - tdSql.close() + tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py index 1b65e38d72587247aabbd7c598c500fa797dfd81..0d10741331578f678198ad8dac597fe56a6c0709 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/sml_json_alltypes.py @@ -83,7 +83,7 @@ class TDTestCase: tdSql.checkData(1, 1, "DOUBLE") tdSql.query("describe db.stb8") if major_ver == "3": - tdSql.checkData(1, 1, "NCHAR") + tdSql.checkData(1, 1, "VARCHAR") tdSql.checkData(1, 2, 16) else: tdSql.checkData(1, 1, "NCHAR") @@ -91,7 +91,7 @@ class TDTestCase: tdSql.query("describe db.stb9") if major_ver == "3": - tdSql.checkData(1, 1, "NCHAR") + tdSql.checkData(1, 1, "VARCHAR") tdSql.checkData(1, 2, 16) else: tdSql.checkData(1, 1, "NCHAR") diff --git a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py index a6c2062d6cefb5f41023ae7c5e1926e05c6dfb83..1ccbb1f7d659d4ce64ba83475997539d3f9d5845 100644 --- a/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py +++ b/tests/develop-test/5-taos-tools/taosdump/taosdumpTestInspect.py @@ -11,24 +11,20 @@ # -*- coding: utf-8 -*- -import sys import os from util.log import * from util.cases import * from util.sql import * from util.dnodes import * -import subprocess class TDTestCase: def caseDescription(self): - ''' + """ case1: [TD-14544] taosdump data inspect - ''' - return + """ def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self.tmpdir = "tmp" @@ -36,44 +32,56 @@ class TDTestCase: def getPath(self, tool="taosdump"): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] + elif "src" in selfPath: + projPath = selfPath[: selfPath.find("src")] + elif "/tools/" in selfPath: + projPath = selfPath[: selfPath.find("/tools/")] + elif "/tests/" in selfPath: + projPath = selfPath[: selfPath.find("/tests/")] else: - projPath = selfPath[:selfPath.find("tests")] + tdLog.info("cannot found %s in path: %s, use system's" % (tool, selfPath)) + projPath = "/usr/local/taos/bin" paths = [] - for root, dirs, files in os.walk(projPath): - if ((tool) in files): + for root, dummy, files in os.walk(projPath): + if (tool) in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): + if "packaging" not in rootRealPath: paths.append(os.path.join(root, tool)) break - if (len(paths) == 0): + if len(paths) == 0: return "" return paths[0] def run(self): - tdSql.prepare(replica=f"{self.replicaVar}") + tdSql.prepare() tdSql.execute("drop database if exists db") - tdSql.execute("create database db days 11 keep 3649 blocks 8 ") + tdSql.execute("create database db keep 3649 ") tdSql.execute("use db") tdSql.execute( - "create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)") + "create table st(ts timestamp, c1 INT, c2 BOOL, c3 TINYINT, c4 SMALLINT, c5 BIGINT, c6 FLOAT, c7 DOUBLE, c8 TIMESTAMP, c9 BINARY(10), c10 NCHAR(10), c11 TINYINT UNSIGNED, c12 SMALLINT UNSIGNED, c13 INT UNSIGNED, c14 BIGINT UNSIGNED) tags(n1 INT, w2 BOOL, t3 TINYINT, t4 SMALLINT, t5 BIGINT, t6 FLOAT, t7 DOUBLE, t8 TIMESTAMP, t9 BINARY(10), t10 NCHAR(10), t11 TINYINT UNSIGNED, t12 SMALLINT UNSIGNED, t13 INT UNSIGNED, t14 BIGINT UNSIGNED)" + ) tdSql.execute( - "create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)") + "create table t1 using st tags(1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)" + ) tdSql.execute( - "insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)") + "insert into t1 values(1640000000000, 1, true, 1, 1, 1, 1.0, 1.0, 1, '1', '一', 1, 1, 1, 1)" + ) tdSql.execute( - "create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)") + "create table t2 using st tags(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" + ) tdSql.execute( - "insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)") + "insert into t2 values(1640000000000, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)" + ) -# sys.exit(1) + # sys.exit(1) - binPath = self.getPath("taosdump") - if (binPath == ""): + binPath = self.getPath() + if binPath == "": tdLog.exit("taosdump not found!") else: tdLog.info("taosdump found in %s" % binPath) @@ -85,35 +93,73 @@ class TDTestCase: os.system("rm -rf %s" % self.tmpdir) os.makedirs(self.tmpdir) - os.system( - "%s --databases db -o %s -T 1" % - (binPath, self.tmpdir)) + os.system("%s --databases db -o %s -T 1" % (binPath, self.tmpdir)) -# sys.exit(1) + # sys.exit(1) - taosdumpInspectCmd = "%s -I %s/*.avro* -s | grep 'Schema:'|wc -l" % ( - binPath, self.tmpdir) - schemaTimes = subprocess.check_output( - taosdumpInspectCmd, shell=True).decode("utf-8") + taosdumpInspectCmd = "%s -I %s/taosdump.*/*.avro* -s | grep 'Schema:'|wc -l" % ( + binPath, + self.tmpdir, + ) + schemaTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode( + "utf-8" + ) print("schema found times: %d" % int(schemaTimes)) - if (int(schemaTimes) != 3): + if int(schemaTimes) != 1: caller = inspect.getframeinfo(inspect.stack()[0][0]) tdLog.exit( - "%s(%d) failed: expected schema found times 3, actual %d" % - (caller.filename, caller.lineno, int(schemaTimes))) + "%s(%d) failed: expected schema found times 1, actual %d" + % (caller.filename, caller.lineno, int(schemaTimes)) + ) + + taosdumpInspectCmd = ( + "%s -I %s/taosdump*/data*/*.avro* -s | grep 'Schema:'|wc -l" + % (binPath, self.tmpdir) + ) + schemaTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode( + "utf-8" + ) + print("schema found times: %d" % int(schemaTimes)) - taosdumpInspectCmd = "%s -I %s/*.avro* | grep '=== Records:'|wc -l" % ( - binPath, self.tmpdir) - recordsTimes = subprocess.check_output( - taosdumpInspectCmd, shell=True).decode("utf-8") + if int(schemaTimes) != 2: + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected schema found times 2, actual %d" + % (caller.filename, caller.lineno, int(schemaTimes)) + ) + + taosdumpInspectCmd = ( + "%s -I %s/taosdump*/*.avro* | grep '=== Records:'|wc -l" + % (binPath, self.tmpdir) + ) + recordsTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode( + "utf-8" + ) + print("records found times: %d" % int(recordsTimes)) + + if int(recordsTimes) != 1: + caller = inspect.getframeinfo(inspect.stack()[0][0]) + tdLog.exit( + "%s(%d) failed: expected records found times 1, actual %d" + % (caller.filename, caller.lineno, int(recordsTimes)) + ) + + taosdumpInspectCmd = ( + "%s -I %s/taosdump*/data*/*.avro* | grep '=== Records:'|wc -l" + % (binPath, self.tmpdir) + ) + recordsTimes = subprocess.check_output(taosdumpInspectCmd, shell=True).decode( + "utf-8" + ) print("records found times: %d" % int(recordsTimes)) - if (int(recordsTimes) != 3): + if int(recordsTimes) != 2: caller = inspect.getframeinfo(inspect.stack()[0][0]) tdLog.exit( - "%s(%d) failed: expected records found times 3, actual %d" % - (caller.filename, caller.lineno, int(recordsTimes))) + "%s(%d) failed: expected records found times 2, actual %d" + % (caller.filename, caller.lineno, int(recordsTimes)) + ) def stop(self): tdSql.close() diff --git a/tests/docs-examples-test/python.sh b/tests/docs-examples-test/python.sh index ccb391b7527fbf2490911d868d08c87436221162..31342b33d7a6a0038bb02c32e2ce99c788957792 100644 --- a/tests/docs-examples-test/python.sh +++ b/tests/docs-examples-test/python.sh @@ -83,4 +83,5 @@ python3 fast_write_example.py # 20 pip3 install kafka-python -python3 kafka_example.py +python3 kafka_example_consumer.py + diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 188ab86944dafe350d72fe0cc8554131ad4c8bbe..a24efeac5fcbdcda9257961d2bbccaead8ce54a4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -82,6 +82,7 @@ ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim ,,y,script,./test.sh -f tsim/parser/alter_column.sim ,,y,script,./test.sh -f tsim/parser/alter_stable.sim @@ -178,6 +179,7 @@ ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/event.sim ,,y,script,./test.sh -f tsim/query/forceFill.sim ,,n,script,./test.sh -f tsim/query/join.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim @@ -249,6 +251,8 @@ ,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim ,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim ,,y,script,./test.sh -f tsim/trans/lossdata1.sim ,,y,script,./test.sh -f tsim/trans/create_db.sim ,,y,script,./test.sh -f tsim/tmq/basic1.sim @@ -426,6 +430,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_manage.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py @@ -662,6 +667,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 + +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 @@ -672,6 +680,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 + ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 ,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 #,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 @@ -1059,6 +1068,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_data.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py @@ -1083,4 +1093,4 @@ ,,n,docs-examples-test,bash node.sh ,,n,docs-examples-test,bash csharp.sh ,,n,docs-examples-test,bash jdbc.sh -#,,n,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash go.sh diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index 8b88a8007d640e6e762b204d4f2a2bde8206d27a..d0086c733e2e3fb7271aafe4f85f4826dcd69a9f 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -70,7 +70,7 @@ mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugNoSan date docker run \ -v $REP_MOUNT_PARAM \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true;make -j || exit 1 " mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 72b7e1fddff7fcfce3c9fb83eeb74f2cefc01f18..2607cf63c29488b05a998fafd1565f29917d8e5e 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -53,7 +53,7 @@ class ConfigureyCluster: # configure dnoe of independent mnodes if num <= self.mnodeNums and self.mnodeNums != 0 and independentMnode == True : - dnode.addExtraCfg("supportVnodes", 0) + dnode.addExtraCfg("supportVnodes", 1024) # print(dnode) self.dnodes.append(dnode) return self.dnodes diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index a1682f47b3e229e24a9b7bdd750b8cbe4890dea5..ddb3bfaa269e90a3272233181e0b7e3feab5bbab 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -15,6 +15,7 @@ import sys import os import os.path import platform +import distro import subprocess from time import sleep import base64 @@ -22,6 +23,7 @@ import json import copy from fabric2 import Connection from util.log import * +from shutil import which class TDSimClient: @@ -166,9 +168,9 @@ class TDDnode: if value: selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): - self.execPath = os.path.abspath(self.path + "/community/tests/script/sh/exec.sh") + self.execPath = os.path.abspath(self.path + "/community/tests/script/sh/exec.sh") else: - self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") + self.execPath = os.path.abspath(self.path + "/tests/script/sh/exec.sh") def getDataSize(self): totalSize = 0 @@ -686,7 +688,7 @@ class TDDnodes: if ("community" in selfPath): self.stopDnodesPath = os.path.abspath(self.path + "/community/tests/script/sh/stop_dnodes.sh") self.stopDnodesSigintPath = os.path.abspath(self.path + "/community/tests/script/sh/sigint_stop_dnodes.sh") - else: + else: self.stopDnodesPath = os.path.abspath(self.path + "/tests/script/sh/stop_dnodes.sh") self.stopDnodesSigintPath = os.path.abspath(self.path + "/tests/script/sh/sigint_stop_dnodes.sh") tdLog.info("run in address sanitizer mode") @@ -765,7 +767,8 @@ class TDDnodes: def stopAll(self): tdLog.info("stop all dnodes, asan:%d" % self.asan) - if self.asan: + distro_id = distro.id() + if self.asan and distro_id != "alpine": tdLog.info("execute script: %s" % self.stopDnodesPath) os.system(self.stopDnodesPath) tdLog.info("execute finished") @@ -777,24 +780,41 @@ class TDDnodes: for i in range(len(self.dnodes)): self.dnodes[i].stop() - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}' | xargs" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - while(processID): - if platform.system().lower() == 'windows': - killCmd = "kill -9 %s > nul 2>&1" % processID - else: - killCmd = "kill -9 %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8").strip() + + if (distro_id == "alpine"): + print(distro_id) + psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + while(processID): + print(processID) + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + else: + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8").strip() + + else: + psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep| grep -v defunct | awk '{print $2}' | xargs" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + if processID: + cmd = "sudo systemctl stop taosd" + os.system(cmd) + # if os.system(cmd) != 0 : + # tdLog.exit(cmd) + psCmd = "ps -ef|grep -w taosd| grep -v grep| grep -v defunct | awk '{print $2}' | xargs" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + while(processID): + if platform.system().lower() == 'windows': + killCmd = "kill -9 %s > nul 2>&1" % processID + else: + killCmd = "kill -9 %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8").strip() if self.killValgrind == 1: psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}' | xargs" diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 9cfd1d368edfa319aa4e80b307cbcc09020245ff..bdf3f20e1546e029dfbf7b41b280e857fb732020 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -261,6 +261,70 @@ class TDSql: tdLog.info(f"sql:{self.sql}, row:{row} col:{col} data:{self.queryResult[row][col]} == expect:{data}") + # return true or false replace exit, no print out + def checkRowColNoExit(self, row, col): + caller = inspect.getframeinfo(inspect.stack()[2][0]) + if row < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if col < 0: + args = (caller.filename, caller.lineno, self.sql, row) + return False + if row > self.queryRows: + args = (caller.filename, caller.lineno, self.sql, row, self.queryRows) + return False + if col > self.queryCols: + args = (caller.filename, caller.lineno, self.sql, col, self.queryCols) + return False + + return True + + + # return true or false replace exit, no print out + def checkDataNoExit(self, row, col, data): + if self.checkRowColNoExit(row, col) == False: + return False + if self.queryResult[row][col] != data: + if self.cursor.istype(col, "TIMESTAMP"): + # suppose user want to check nanosecond timestamp if a longer data passed + if (len(data) >= 28): + if pd.to_datetime(self.queryResult[row][col]) == pd.to_datetime(data): + return True + else: + if self.queryResult[row][col] == _parse_datetime(data): + return True + return False + + if str(self.queryResult[row][col]) == str(data): + return True + elif isinstance(data, float): + if abs(data) >= 1 and abs((self.queryResult[row][col] - data) / data) <= 0.000001: + return True + elif abs(data) < 1 and abs(self.queryResult[row][col] - data) <= 0.000001: + return True + else: + return False + else: + return False + + return True + + + # loop execute sql then sleep(waitTime) , if checkData ok break loop + def checkDataLoop(self, row, col, data, sql, loopCount, waitTime): + # loop check util checkData return true + for i in range(loopCount): + self.query(sql) + if self.checkDataNoExit(row, col, data) : + self.checkData(row, col, data) + return + time.sleep(waitTime) + + # last check + self.query(sql) + self.checkData(row, col, data) + + def getData(self, row, col): self.checkRowCol(row, col) return self.queryResult[row][col] diff --git a/tests/pytest/util/taosadapter.py b/tests/pytest/util/taosadapter.py index 7dd62266319bd1462ce7bad6ff6ec358cfe31678..79ea86c5bd820c1327cf844b35eb8800fefdfe60 100644 --- a/tests/pytest/util/taosadapter.py +++ b/tests/pytest/util/taosadapter.py @@ -238,14 +238,13 @@ class TAdapter: if self.running != 0: psCmd = f"ps -ef|grep -w {toBeKilled}| grep -v grep | awk '{{print $2}}'" + # psCmd = f"pgrep {toBeKilled}" processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() - # psCmd = f"pgrep {toBeKilled}" - - while(processID): - killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) + while(processID): + killCmd = "kill %s %s > /dev/null 2>&1" % (signal, processID) os.system(killCmd) time.sleep(1) - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() if not platform.system().lower() == 'windows': port = 6041 fuserCmd = f"fuser -k -n tcp {port} > /dev/null" diff --git a/tests/requirements.txt b/tests/requirements.txt index ce459414c44c6adcb8c8a77b03ea790480a41cd2..c6e27fd3be6b35c268c8f7b7b97da048224561e3 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,3 +3,8 @@ numpy fabric2 psutil pandas +toml +distro +requests +pexpect +faker diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index d1a80d9683a8599ba4123236dc8738c84f0bc291..0903095dc98dcf3652546859003fa65fd88e1569 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -232,8 +232,8 @@ CaseCtrl gCaseCtrl = { .printCreateTblSql = true, .printQuerySql = true, .printStmtSql = true, - .printVerbose = false, - .printRes = false, + .printVerbose = true, + .printRes = true, .autoCreateTbl = false, .numericParam = false, .rowNum = 0, diff --git a/tests/script/coverage_test.sh b/tests/script/coverage_test.sh index e0e7490d3ab7616cfc44ceab7c4c951b21663004..3f99dde5442c5f0c9e88b5ea1df37f7a64e54003 100755 --- a/tests/script/coverage_test.sh +++ b/tests/script/coverage_test.sh @@ -130,6 +130,9 @@ function runSimCases() { function runPythonCases() { echo "=== Run python cases ===" + + cd $TDENGINE_DIR/tests/parallel_test + sed -i '/compatibility.py/d' cases.task cd $TDENGINE_DIR/tests/system-test runCasesOneByOne ../parallel_test/cases.task system-test diff --git a/tests/script/sh/cfg.sh b/tests/script/sh/cfg.sh index 7d4d747e54bab9ed98b42ab1902f8580e17fca6c..bf58185f6201b5046608881d9c9f1a5b5e8de3ac 100755 --- a/tests/script/sh/cfg.sh +++ b/tests/script/sh/cfg.sh @@ -45,11 +45,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 2bd4eaa5484dedde92974ea6cea4f65e6d8aed91..b6cb4f6280b24453a9a8171ace944b4003948b53 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -2,7 +2,19 @@ set +e #set -x +if [[ "$OSTYPE" == "darwin"* ]]; then + TD_OS="Darwin" +else + OS=$(cat /etc/*-release | grep "^NAME=" | cut -d= -f2) + len=$(echo ${#OS}) + len=$((len-2)) + TD_OS=$(echo -ne ${OS:1:${len}} | cut -d" " -f1) +fi +if [[ "$TD_OS" == "Alpine" ]]; then + echo -e "os is Alpine,skip check Asan" + exit 0 +fi unset LD_PRELOAD SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ @@ -40,6 +52,9 @@ python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # /root/TDengine/source/common/src/tdataformat.c:1876:7: runtime error: signed integer overflow: 8252423483843671206 + 2406154664059062870 cannot be represented in type 'long int' # /home/chr/TDengine/source/libs/scalar/src/filter.c:3149:14: runtime error: applying non-zero offset 18446744073709551615 to null pointer +# /home/chr/TDengine/source/libs/scalar/src/filter.c:3149:14: runtime error: applying non-zero offset 18446744073709551615 to null pointer +# /home/TDinternal/community/source/libs/scalar/src/sclvector.c:1109:66: runtime error: signed integer overflow: 9223372034707292160 + 1676867897049 cannot be represented in type 'long int' + runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "signed integer overflow" |grep -v "strerror.c"| grep -v "asan_malloc_linux.cc" |wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" diff --git a/tests/script/sh/clear.sh b/tests/script/sh/clear.sh index 4ee296cf058370552b14c31b67006830e84847dd..587a33f633fbfac4319cf5f6724a29fa60680a91 100755 --- a/tests/script/sh/clear.sh +++ b/tests/script/sh/clear.sh @@ -48,11 +48,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 217bd66ef61d937a575145e36c645db37b20f046..7da8da09bfcf3810b692ff25f37333c93ae87497 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -39,11 +39,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/exec-default.sh b/tests/script/sh/exec-default.sh index f648315c6745f63cfba9eb06ecfd53a9ccef1fed..0a83fa1dc81e5b510e85c2a5e2bf7f646f69e684 100755 --- a/tests/script/sh/exec-default.sh +++ b/tests/script/sh/exec-default.sh @@ -54,11 +54,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/exec-no-random-fail.sh b/tests/script/sh/exec-no-random-fail.sh index e01b18a8e6bb0d4631a28c64c6accd57bceb9076..b15783afb569a760405091dea54dcbb6832fa920 100755 --- a/tests/script/sh/exec-no-random-fail.sh +++ b/tests/script/sh/exec-no-random-fail.sh @@ -54,11 +54,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/exec-random-fail.sh b/tests/script/sh/exec-random-fail.sh index 1f31899e3ac26861fc1860d10cb0a4899ff7bf36..15f2864ca6d9b371412e173ea81034a52a437994 100755 --- a/tests/script/sh/exec-random-fail.sh +++ b/tests/script/sh/exec-random-fail.sh @@ -54,11 +54,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index c8cb121b8a88b618cbd7410e95741091655cef38..f548a4cc418d34ac5d4fe27fb944181c54b3a5c2 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -10,6 +10,14 @@ set +e #set -x +if [[ "$OSTYPE" == "darwin"* ]]; then + TD_OS="Darwin" +else + OS=$(cat /etc/*-release | grep "^NAME=" | cut -d= -f2) + len=$(echo ${#OS}) + len=$((len-2)) + TD_OS=$(echo -ne ${OS:1:${len}} | cut -d" " -f1) +fi unset LD_PRELOAD UNAME_BIN=`which uname` @@ -44,7 +52,10 @@ do ;; esac done - +if [[ "$VALGRIND_OPTION" = "true" ]] && [[ "$TD_OS" == "Alpine" ]]; then + echo alpine skip valgrind + VALGRIND_OPTION="false" +fi SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR/../ SCRIPT_DIR=`pwd` @@ -59,11 +70,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/exec_tarbitrator.sh b/tests/script/sh/exec_tarbitrator.sh index e985bd65856025b2db8dfef724fdc652b2a03392..eacf5fd2262e54e5ef8d2f475e9cc4faf0515430 100755 --- a/tests/script/sh/exec_tarbitrator.sh +++ b/tests/script/sh/exec_tarbitrator.sh @@ -51,11 +51,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/sh/mv_old_data.sh b/tests/script/sh/mv_old_data.sh index 3f4be6714fd757ee60d21ee97be5d411e3f95bdd..1a6bd22c621873253bf98690900759c1ef2304d7 100755 --- a/tests/script/sh/mv_old_data.sh +++ b/tests/script/sh/mv_old_data.sh @@ -20,11 +20,7 @@ fi TAOS_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/test.sh b/tests/script/test.sh index 19180382fd30de26667fef44a5ebc54d19d5b6a7..b10865dd65a8ac2a0771ca49b58ef72ed473bec3 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -51,11 +51,7 @@ fi TOP_DIR=`pwd` TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` diff --git a/tests/script/tsim/dnode/balance3.sim b/tests/script/tsim/dnode/balance3.sim index 2fb284b46614e37ee6ac27bd13a16eb7429e9d86..d0235e096e483e93c1785b555be93150c2a3c35e 100644 --- a/tests/script/tsim/dnode/balance3.sim +++ b/tests/script/tsim/dnode/balance3.sim @@ -130,13 +130,13 @@ print dnode4 openVnodes $data(4)[2] if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 1 then +if $data(2)[2] != 2 then return -1 endi if $data(3)[2] != 2 then return -1 endi -if $data(4)[2] != 2 then +if $data(4)[2] != 1 then return -1 endi @@ -216,7 +216,7 @@ endi print ========== step4 sql create database d3 replica 3 vgroups 1 -sql create table d3.t3 (t timestamp, i int) +sql create table d3.t3 (t timestamp, i int) sql insert into d3.t3 values(now+1s, 35) sql insert into d3.t3 values(now+2s, 34) sql insert into d3.t3 values(now+3s, 33) @@ -231,10 +231,10 @@ print dnode5 openVnodes $data(5)[2] if $data(1)[2] != 2 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 3 then return -1 endi -if $data(4)[2] != 3 then +if $data(4)[2] != 2 then return -1 endi if $data(5)[2] != 2 then @@ -315,10 +315,10 @@ endi if $data(3)[2] != null then return -1 endi -if $data(4)[2] != 2 then +if $data(4)[2] != 3 then return -1 endi -if $data(5)[2] != 3 then +if $data(5)[2] != 2 then return -1 endi if $data(6)[2] != 2 then diff --git a/tests/script/tsim/dnode/balancex.sim b/tests/script/tsim/dnode/balancex.sim index 0cfc64a954f10457ab720990c758cfe4032e900a..6f077be521e4a4835b3c778a1dee25ea61a8c4eb 100644 --- a/tests/script/tsim/dnode/balancex.sim +++ b/tests/script/tsim/dnode/balancex.sim @@ -145,10 +145,10 @@ print dnode2 openVnodes $data(4)[2] if $data(1)[2] != 1 then return -1 endi -if $data(2)[2] != 1 then +if $data(2)[2] != 2 then return -1 endi -if $data(3)[2] != 2 then +if $data(3)[2] != 1 then return -1 endi if $data(4)[2] != 1 then diff --git a/tests/script/tsim/dnode/drop_dnode_force.sim b/tests/script/tsim/dnode/drop_dnode_force.sim index 26e48933be4929e158a407fd512734720be32d95..425e5f48b2f278f3ef2f2bad51b25f81b8133fc2 100644 --- a/tests/script/tsim/dnode/drop_dnode_force.sim +++ b/tests/script/tsim/dnode/drop_dnode_force.sim @@ -120,9 +120,9 @@ if $rows != 12 then return -1 endi -print =============== step3: create qnode snode on dnode 2 -sql create qnode on dnode 2 -sql create snode on dnode 2 +print =============== step3: create qnode snode on dnode 3 +sql create qnode on dnode 3 +sql create snode on dnode 3 sql select * from information_schema.ins_qnodes if $rows != 1 then return -1 @@ -136,16 +136,16 @@ print =============== step4: create mnode on dnode 2 sql create mnode on dnode 3 sql create mnode on dnode 2 $x = 0 -step4: +step4: $x = $x + 1 sleep 1000 if $x == 10 then return -1 endi sql select * from information_schema.ins_mnodes -x step4 -print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] -print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[1][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[1][4] #if $data(1)[2] != leader then # goto step4 #endi @@ -157,7 +157,7 @@ endi #endi print =============== step5: create dnode 5 -system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT system sh/exec.sh -n dnode5 -s start $x = 0 step5: @@ -179,10 +179,10 @@ endi if $data(1)[4] != ready then goto step5 endi -if $data(2)[4] != offline then +if $data(2)[4] != ready then goto step5 endi -if $data(3)[4] != ready then +if $data(3)[4] != offline then goto step5 endi if $data(4)[4] != ready then @@ -192,9 +192,9 @@ if $data(5)[4] != ready then goto step5 endi -print =============== step5a: drop dnode 2 -sql_error drop dnode 2 -sql drop dnode 2 force +print =============== step5a: drop dnode 3 +sql_error drop dnode 3 +sql drop dnode 3 force print select * from information_schema.ins_dnodes; sql select * from information_schema.ins_dnodes; diff --git a/tests/script/tsim/dnode/split_vgroup_replica1.sim b/tests/script/tsim/dnode/split_vgroup_replica1.sim index 1bdd322714a1e8e7443d337daacc69d3ce2a4792..51d63d25f482985b67ae37f2d4f005b8010f759e 100644 --- a/tests/script/tsim/dnode/split_vgroup_replica1.sim +++ b/tests/script/tsim/dnode/split_vgroup_replica1.sim @@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start sql connect -sql create user u1 pass 'taosdata' print =============== step1 create dnode2 sql create dnode $hostname port 7200 @@ -73,8 +72,21 @@ print =============== step3: create database sql use d1 sql create table d1.st (ts timestamp, i int) tags (j int) sql create table d1.c1 using st tags(1) +sql create table d1.c2 using st tags(2) +sql create table d1.c3 using st tags(3) +sql create table d1.c4 using st tags(4) +sql create table d1.c5 using st tags(5) +sql insert into d1.c1 values (now, 1); +sql insert into d1.c2 values (now, 2); +sql insert into d1.c3 values (now, 3); +sql insert into d1.c4 values (now, 4); +sql insert into d1.c5 values (now, 5); sql show d1.tables -if $rows != 1 then +if $rows != 5 then + return -1 +endi +sql select * from d1.st +if $rows != 5 then return -1 endi @@ -82,6 +94,34 @@ print =============== step4: split print split vgroup 2 sql split vgroup 2 +print =============== step5: check split result +sql show d1.tables +#if $rows != 5 then +# return -1 +#endi +#sql select * from d1.st +#if $rows != 5 then +# return -1 +#endi + +print =============== step6: create tables +sql create table d1.c6 using st tags(6) +sql create table d1.c7 using st tags(7) +sql create table d1.c8 using st tags(8) +sql create table d1.c9 using st tags(9) +sql insert into d1.c6 values (now, 6); +sql insert into d1.c7 values (now, 7); +sql insert into d1.c8 values (now, 8); +sql insert into d1.c9 values (now, 9); +sql show d1.tables +#if $rows != 9 then +# return -1 +#endi +#sql select * from d1.st +#if $rows != 9 then +# return -1 +#endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/vnode_clean.sim b/tests/script/tsim/dnode/vnode_clean.sim index ba1d083c68e069fdbfecbd13144f31750a81eb81..aead25938ec0b7c1e52ed69ddbfcd58f79d060b7 100644 --- a/tests/script/tsim/dnode/vnode_clean.sim +++ b/tests/script/tsim/dnode/vnode_clean.sim @@ -207,10 +207,10 @@ print dnode2 openVnodes $data(4)[2] if $data(1)[2] != 1 then return -1 endi -if $data(3)[2] != 1 then +if $data(3)[2] != 2 then return -1 endi -if $data(4)[2] != 2 then +if $data(4)[2] != 1 then return -1 endi diff --git a/tests/script/tsim/insert/update2.sim b/tests/script/tsim/insert/update2.sim new file mode 100644 index 0000000000000000000000000000000000000000..d2b5b052678a93499157dea219571df6ed67c6b7 --- /dev/null +++ b/tests/script/tsim/insert/update2.sim @@ -0,0 +1,221 @@ +################################################################################################ +# migrate from 2.0 insert_update2.sim +################################################################################################ + +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +### 4096*0.8 - 1 = 3275 +$rowSuperBlk = 3275 +### 4096 - 3275 -1 - 1 = 819 +$rowSubBlk = 819 +$ts0 = 1672372800000 +$ts1 = 1672372900000 +$ts2 = 1672686800000 + +$i = 0 +$db = db0 +$stb1 = stb1 +$tb1 = tb1 +$stb2 = stb2 +$tb2 = tb2 + +print ====== create database +sql drop database if exists $db +sql create database $db keep 1000 duration 10 +print ====== create tables +sql use $db +sql create table $stb1 (ts timestamp, c1 bigint, c2 bigint, c3 bigint) tags(t1 int) +sql create table $stb2 (ts timestamp, c1 bigint, c2 bigint, c3 bigint, c4 bigint, c5 bigint, c6 bigint, c7 bigint, c8 bigint, c9 bigint, c10 bigint, c11 bigint, c12 bigint, c13 bigint, c14 bigint, c15 bigint, c16 bigint, c17 bigint, c18 bigint, c19 bigint, c20 bigint, c21 bigint, c22 bigint, c23 bigint, c24 bigint, c25 bigint, c26 bigint, c27 bigint, c28 bigint, c29 bigint, c30 bigint) tags(t1 int) +sql create table $tb1 using $stb1 tags(1) +sql create table $tb2 using $stb2 tags(2) +print ====== tables created + + +print ========== step 1: merge dataRow in mem + +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb1 values ( $ts0 , 1,NULL,0) +sql insert into $tb1 (ts,c2,c3) values ( $ts0 , 1,1) + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +print ========== step 2: merge kvRow in mem +$i = 0 +while $i < $rowSuperBlk + $xs = $i * 10 + $ts = $ts2 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +sql insert into $tb2 (ts,c3,c8,c10) values ( $ts0 , 1,NULL,0) +sql insert into $tb2 (ts,c8,c10) values ( $ts0 , 1,1) + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +print ========== step 3: merge dataRow in file +sql insert into $tb1 (ts,c1) values ( $ts0 , 2) +print ========== step 4: merge kvRow in file +sql insert into $tb2 (ts,c3) values ( $ts0 , 2) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 2 then + return -1 +endi +if $data03 != 1 then + return -1 +endi +if $data04 != 1 then + return -1 +endi + + +print ========== step 5: merge dataRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb1 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw +print ========== step 6: merge kvRow in file/mem +$i = 0 +while $i < $rowSubBlk + $xs = $i * 1 + $ts = $ts1 + $xs + sql insert into $tb2 (ts,c1) values ( $ts , $i ) + $i = $i + 1 +endw + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + +sql insert into $tb1 (ts,c3) values ( $ts0 , 3) +sql insert into $tb2 (ts,c3) values ( $ts0 , 3) +$tsN = $ts0 + 1 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) +$tsN = $ts0 + 2 +sql insert into $tb1 (ts,c1,c3) values ( $tsN , 1,0) +sql insert into $tb2 (ts,c3,c8) values ( $tsN , 100,200) + +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 2000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sleep 2000 + + +sql select * from $tb1 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data02 != 1 then + return -1 +endi +if $data03 != 3 then + return -1 +endi + +sql select ts,c1,c3,c8,c10 from $tb2 where ts = $ts0 +if $rows != 1 then + return -1 +endi +if $data01 != NULL then + return -1 +endi +if $data02 != 3 then + return -1 +endi +if $data03 != 1 then + return -1 +endi + +if $data04 != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/event.sim b/tests/script/tsim/query/event.sim new file mode 100644 index 0000000000000000000000000000000000000000..16fe0f4a137426a9ef6b9bf0fd0434290584e581 --- /dev/null +++ b/tests/script/tsim/query/event.sim @@ -0,0 +1,239 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== prepare data + +sql drop database if exists db1; +sql create database db1 vgroups 5; +sql use db1; +sql create stable sta (ts timestamp, f1 int, f2 binary(10), f3 bool) tags(t1 int, t2 bool, t3 binary(10)); +sql create table tba1 using sta tags(0, false, '0'); +sql create table tba2 using sta tags(1, true, '1'); +sql insert into tba1 values ('2022-09-26 15:15:01', 0, "a", false); +sql insert into tba1 values ('2022-09-26 15:15:02', 1, "0", true); +sql insert into tba1 values ('2022-09-26 15:15:03', 5, "5", false); +sql insert into tba1 values ('2022-09-26 15:15:04', 3, 'b', false); +sql insert into tba1 values ('2022-09-26 15:15:05', 0, '1', false); +sql insert into tba1 values ('2022-09-26 15:15:06', 2, 'd', true); +sql insert into tba2 values ('2022-09-26 15:15:01', 0, "a", false); +sql insert into tba2 values ('2022-09-26 15:15:02', 1, "0", true); +sql insert into tba2 values ('2022-09-26 15:15:03', 5, "5", false); +sql insert into tba2 values ('2022-09-26 15:15:04', 3, 'b', false); +sql insert into tba2 values ('2022-09-26 15:15:05', 0, '1', false); +sql insert into tba2 values ('2022-09-26 15:15:06', 2, 'd', true); + +# child table: no window +print ====> select count(*) from tba1 event_window start with f1 = 0 end with f2 = 'c'; +sql select count(*) from tba1 event_window start with f1 = 0 end with f2 = 'c'; +if $rows != 0 then + return -1 +endi + +# child table: single row window +print ====> select count(*) from tba1 event_window start with f1 = 0 end with f3 = false; +sql select count(*) from tba1 event_window start with f1 = 0 end with f3 = false; +if $rows != 2 then + return -1 +endi +if $data00 != 1 then + return -1 +endi + +# child table: multi rows window +print ====> select count(*) from tba1 event_window start with f1 = 0 end with f2 = 'b'; +sql select count(*) from tba1 event_window start with f1 = 0 end with f2 = 'b'; +if $rows != 1 then + return -1 +endi +if $data00 != 4 then + return -1 +endi + +# child table: multi windows +print ====> select count(*) from tba1 event_window start with f1 >= 0 end with f3 = true; +sql select count(*) from tba1 event_window start with f1 >= 0 end with f3 = true; +if $rows != 2 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 4 then + return -1 +endi + +# super table: no window +print ====> select count(*) from sta event_window start with f1 = 0 end with f2 = 'c'; +sql select count(*) from sta event_window start with f1 = 0 end with f2 = 'c'; +if $rows != 0 then + return -1 +endi + +# super table: single row window +print ====> select count(*) from sta event_window start with f1 = 0 end with f3 = false; +sql select count(*) from sta event_window start with f1 = 0 end with f3 = false; +if $rows != 4 then + return -1 +endi +if $data00 != 1 then + return -1 +endi + +# super table: multi rows window +print ====> select count(*) from sta event_window start with f1 = 0 end with f2 = 'b'; +sql select count(*) from sta event_window start with f1 = 0 end with f2 = 'b'; +if $rows != 1 then + return -1 +endi +if $data00 != 7 then + return -1 +endi + +# super table: multi windows +print ====> select count(*) from sta event_window start with f1 >= 0 end with f3 = true; +sql select count(*) from sta event_window start with f1 >= 0 end with f3 = true; +if $rows != 4 then + return -1 +endi +if $data00 != 3 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != 7 then + return -1 +endi +if $data30 != 1 then + return -1 +endi + +# multi-child table: no window +print ====> select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f2 = 'c'; +sql select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f2 = 'c'; +if $rows != 0 then + return -1 +endi + +# multi-child table: single row window +print ====> select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f3 = false; +sql select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f3 = false; +if $rows != 4 then + return -1 +endi +if $data01 != 1 then + return -1 +endi + +# multi-child table: multi rows window +print ====> select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f2 = 'b'; +sql select tbname, count(*) from sta partition by tbname event_window start with f1 = 0 end with f2 = 'b'; +if $rows != 2 then + return -1 +endi +if $data01 != 4 then + return -1 +endi +if $data11 != 4 then + return -1 +endi + +# multi-child table: multi windows +print ====> select tbname, count(*) from sta partition by tbname event_window start with f1 >= 0 end with f3 = true; +sql select tbname, count(*) from sta partition by tbname event_window start with f1 >= 0 end with f3 = true; +if $rows != 4 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data11 != 4 then + return -1 +endi + +# where + partition by +print ====> select tbname, count(*) from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0; +sql select tbname, count(*) from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0; +if $rows != 4 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data11 != 2 then + return -1 +endi + +# where + order by +print ====> select count(*) cnt from tba1 where f3 = false event_window start with f1 >0 end with f2 > 0 order by cnt desc; +sql select count(*) cnt from tba1 where f3 = false event_window start with f1 >0 end with f2 > 0 order by cnt desc; +if $rows != 2 then + return -1 +endi +if $data00 != 2 then + return -1 +endi +if $data10 != 1 then + return -1 +endi + +# where + partition by + order by +print ====> select tbname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt; +sql select tbname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt; +if $rows != 4 then + return -1 +endi +if $data01 != 1 then + return -1 +endi +if $data11 != 1 then + return -1 +endi + +# where + partition by + order by + limit +print ====> select tbname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2; +sql select tbname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2; +if $rows != 2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data11 != 2 then + return -1 +endi + +# subquery(where + partition by + order by + limit) +print ====> select * from (select tbname tname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2); +sql select * from (select tbname tname, count(*) cnt from sta where f3 = false partition by tbname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2); +if $rows != 2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data11 != 2 then + return -1 +endi + +# subquery + where + partition by + order by + limit +print ====> select tname, count(*) cnt from (select tbname tname, * from sta) where f3 = false partition by tname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2; +sql select tname, count(*) cnt from (select tbname tname, * from sta) where f3 = false partition by tname event_window start with f1 >0 end with f2 > 0 order by cnt limit 2 offset 2; +if $rows != 2 then + return -1 +endi +if $data01 != 2 then + return -1 +endi +if $data11 != 2 then + return -1 +endi + +sql_error select f1, f2 from sta event_window start with f1 >0 end with f2 > 0; +sql_error select count(*) from sta event_window start with f1 >0 end with f2 > 0 partition by tbname; +sql_error select count(*) from sta event_window start with f1 >0 end with f2 > 0 group by tbname; +sql_error select count(*) from sta event_window start with f1 >0 end with f2 > 0 fill(NULL); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/explain.sim b/tests/script/tsim/query/explain.sim index 78d905233b7d59d64f2c17cc5a97b7b84888bfab..eb6b102bd9a8d85352dc1312d906fcbabb22b960 100644 --- a/tests/script/tsim/query/explain.sim +++ b/tests/script/tsim/query/explain.sim @@ -29,43 +29,62 @@ sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..2 #sql create table tb4 using st2 tags(4); #sql insert into tb4 values (now, 4, "Bitmap Heap Scan on tenk1 t1 (cost=5.07..229.20 rows=101 width=244) (actual time=0.080..0.526 rows=100 loops=1)"); +# for explain insert into select +sql create table t1 (ts timestamp, f1 int, f2 binary(200), t1 int); print ======== step2 sql explain select * from st1 where -2; +sql explain insert into t1 select * from st1 where -2; sql explain select ts from tb1; +sql explain insert into t1(ts) select ts from tb1; sql explain select * from st1; +sql explain insert into t1 select * from st1; sql explain select * from st1 order by ts; +sql explain insert into t1 select * from st1 order by ts; sql explain select * from information_schema.ins_stables; sql explain select count(*),sum(f1) from tb1; sql explain select count(*),sum(f1) from st1; sql explain select count(*),sum(f1) from st1 group by f1; #sql explain select count(f1) from tb1 interval(10s, 2s) sliding(3s) fill(prev); +sql explain insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s); print ======== step3 sql explain verbose true select * from st1 where -2; +sql explain verbose true insert into t1 select * from st1 where -2; sql explain verbose true select ts from tb1 where f1 > 0; +sql explain verbose true insert into t1(ts) select ts from tb1 where f1 > 0; sql explain verbose true select * from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; +sql explain verbose true insert into t1 select * from st1 where f1 > 0 and ts > '2020-10-31 00:00:00' and ts < '2021-10-31 00:00:00'; sql explain verbose true select count(*) from st1 partition by tbname slimit 1 soffset 2 limit 2 offset 1; sql explain verbose true select * from information_schema.ins_stables where db_name='db2'; sql explain verbose true select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0; +sql explain verbose true insert into t1(ts) select st1.f1 from st1 join st2 on st1.ts=st2.ts and st1.f1 > 0; +sql explain verbose true insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s); print ======== step4 sql explain analyze select ts from st1 where -2; +sql explain analyze insert into t1(ts) select ts from st1 where -2; sql explain analyze select ts from tb1; +sql explain analyze insert into t1(ts) select ts from tb1; sql explain analyze select ts from st1; -sql explain analyze select ts from st1; +sql explain analyze insert into t1(ts) select ts from st1; sql explain analyze select ts from st1 order by ts; +sql explain analyze insert into t1(ts) select ts from st1 order by ts; sql explain analyze select * from information_schema.ins_stables; sql explain analyze select count(*),sum(f1) from tb1; sql explain analyze select count(*),sum(f1) from st1; sql explain analyze select count(*),sum(f1) from st1 group by f1; +sql explain analyze insert into t1(ts, t1) select _wstart, count(*) from st1 interval(10s); print ======== step5 sql explain analyze verbose true select ts from st1 where -2; +sql explain analyze verbose true insert into t1(ts) select ts from st1 where -2; sql explain analyze verbose true select ts from tb1; +sql explain analyze verbose true insert into t1(ts) select ts from tb1; sql explain analyze verbose true select ts from st1; -sql explain analyze verbose true select ts from st1; +sql explain analyze verbose true insert into t1(ts) select ts from st1; sql explain analyze verbose true select ts from st1 order by ts; +sql explain analyze verbose true insert into t1(ts) select ts from st1 order by ts; sql explain analyze verbose true select * from information_schema.ins_stables; sql explain analyze verbose true select count(*),sum(f1) from tb1; sql explain analyze verbose true select count(*),sum(f1) from st1; diff --git a/tests/script/tsim/query/sys_tbname.sim b/tests/script/tsim/query/sys_tbname.sim index 4587dcd4f7ff9c455f8f87c663e7f00cbf32957a..7b3953129a81d6fa56b093c8b048b2071b6fbe70 100644 --- a/tests/script/tsim/query/sys_tbname.sim +++ b/tests/script/tsim/query/sys_tbname.sim @@ -53,7 +53,7 @@ endi sql select tbname from information_schema.ins_tables; print $rows $data00 -if $rows != 32 then +if $rows != 33 then return -1 endi if $data00 != @ins_tables@ then diff --git a/tests/script/tsim/scalar/tsConvert.sim b/tests/script/tsim/scalar/tsConvert.sim index 75efdb442f5055b559950173c68941a2260aa5bd..ad8d3b3b93382ac706edaeac85ab58fabf14e04d 100644 --- a/tests/script/tsim/scalar/tsConvert.sim +++ b/tests/script/tsim/scalar/tsConvert.sim @@ -13,7 +13,7 @@ sql insert into tb1 values ('2022-07-10 16:31:00', 1, '1'); sql insert into tb1 values ('2022-07-10 16:32:00', 2, '2'); sql insert into tb1 values ('2022-07-10 16:33:00', 3, '3'); sql insert into tb1 values ('2022-07-10 16:34:00', 4, '4'); -sql select * from (select ts,TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),timediff(TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),1h) as td from tb1 where ts >='2022-06-01 00:00:00' and ts <=' 2022-11-3 23:59:59' ) t where td >12; +sql select * from (select ts,TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),timediff(TIMETRUNCATE(ts,1d),TIMETRUNCATE(ts,1h),1h) as td from tb1 where ts >='2022-06-01 00:00:00' and ts <='2022-11-3 23:59:59' ) t where td >12; if $rows != 4 then return -1 endi diff --git a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim index 508e6f88c14f09d4e55c8bd3d40ea78426e5fa43..b3144e4e0dd217319a0d58bf3222360fcd5fa355 100644 --- a/tests/script/tsim/sma/rsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/rsmaCreateInsertQuery.sim @@ -81,8 +81,60 @@ if $data01 != 10 then endi #=================================================================== +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print =============== wait maxdelay 15+2 seconds for results after reboot +sleep 17000 + +print =============== select * from retention level 2 from memory after reboot +sql select * from ct1; +print $data00 $data01 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + + +if $data01 != 1 then + if $data01 != 10 then + print =============> $data01 + print retention level 2 file result $data01 != 1 or 10 + return -1 + endi +endi + +print =============== select * from retention level 1 from memory after reboot +sql select * from ct1 where ts > now-8d; +print $data00 $data01 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 1 then + if $data01 != 10 then + print retention level 1 file result $data01 != 1 or 10 + return -1 + endi +endi + +print =============== select * from retention level 0 from memory after reboot +sql select * from ct1 where ts > now-3d; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi -#==================== reboot to trigger commit data to file +#==================== flush database to trigger commit data to file sql flush database d0; system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index 4117a2403dcc7156a8b9ea3adb840fe25b26b376..0c9d23335ea3b378a2dbe849dc8f7fd773470dea 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -82,9 +82,62 @@ if $data01 != 10 then endi #=================================================================== +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +print =============== wait maxdelay 5+2 seconds for results after reboot +sleep 7000 + +print =============== select * from retention level 2 from memory after reboot +sql select * from ct1; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 2 file rows $rows > 2 + return -1 +endi + + +if $data01 != 100 then + if $data01 != 10 then + print retention level 2 file result $data01 != 100 or 10 + return -1 + endi +endi + +print =============== select * from retention level 1 from memory after reboot +sql select * from ct1 where ts > now-8d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows > 2 then + print retention level 1 file rows $rows > 2 + return -1 +endi + +if $data01 != 100 then + if $data01 != 10 then + print retention level 1 file result $data01 != 100 or 10 + return -1 + endi +endi + +print =============== select * from retention level 0 from memory after reboot +sql select * from ct1 where ts > now-3d; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 + +if $rows < 1 then + print retention level 0 file rows $rows < 1 + return -1 +endi + +if $data01 != 10 then + print retention level 0 file result $data01 != 10 + return -1 +endi -#==================== reboot to trigger commit data to file +#==================== flush database to trigger commit data to file sql flush database d0; system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim index 442b4970e4e09652666378962a8e04d9ae8616fd..9b6fc96bc00fde8418bf1d29d4c8cd8fc7f02091 100644 --- a/tests/script/tsim/sma/tsmaCreateInsertQuery.sim +++ b/tests/script/tsim/sma/tsmaCreateInsertQuery.sim @@ -7,9 +7,10 @@ sql connect print =============== create database sql create database d1 keep 36500d vgroups 1 sql use d1 +sql alter local 'querySmaOptimize' '1'; print =============== create super table, include column type for count/sum/min/max/first -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 double, c4 binary(10),c5 nchar(10)) tags (t1 int unsigned) sql show stables if $rows != 1 then @@ -25,16 +26,32 @@ if $rows != 1 then endi print =============== insert data, mode1: one row one table in sql -sql insert into ct1 values('2022-10-19 09:55:45.682', 10, 2.0, 3.0) -sql insert into ct1 values('2022-10-19 09:55:46.682', 11, 2.1, 3.1)('2022-10-19 09:55:47.682', -12, -2.2, -3.2)('2022-10-19 09:55:48.682', -13, -2.3, -3.3) +sql insert into ct1 values('2022-10-19 09:55:45.682', 10, 2.0, 3.0, "a", "n0") +sql insert into ct1 values('2022-10-19 09:55:46.682', 11, 2.1, 3.1,"b","n1")('2022-10-19 09:55:47.682', -12, -2.2, -3.2,"c","n2")('2022-10-19 09:55:48.682', -13, -2.3, -3.3,"d","n3") print =============== create sma index from super table sql create sma index sma_index_name1 on stb function(max(c1),max(c2),min(c1)) interval(5m,10s) sliding(5m) watermark 1s max_delay 1s -print $data00 $data01 $data02 $data03 +sql create sma index sma_index_name2 on stb function(sum(c1),first(c1), last(c1), first(c4),last(c4),count(c4),first(c5),last(c5),count(c5),apercentile(c1,80,"t-digest"), avg(c2),count(c3), spread(c3), stddev(c2), hyperloglog(c2), hyperloglog(c4), hyperloglog(c5)) interval(5m,10s) sliding(5m); +# for varchar/binary +sql_error create sma index sma_index_name3 on stb function(sum(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(min(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(max(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(avg(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(apercentile(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(spread(c4)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(stddev(c4)) interval(5m,10s) sliding(5m); +# for nchar +sql_error create sma index sma_index_name3 on stb function(sum(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(min(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(max(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(avg(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(apercentile(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(spread(c5)) interval(5m,10s) sliding(5m); +sql_error create sma index sma_index_name3 on stb function(stddev(c5)) interval(5m,10s) sliding(5m); print =============== trigger stream to execute sma aggr task and insert sma data into sma store -sql insert into ct1 values('2022-10-19 09:55:50.682', 20, 20.0, 30.0) +sql insert into ct1 values('2022-10-19 09:55:50.682', 20, 20.0, 30.0,"e","n5") #==================== sleep 2s to wait for tsma result sleep 2000 @@ -42,9 +59,11 @@ print =============== show streams ================================ sql show streams; print $data00 $data01 $data02 -if $data00 != sma_index_name1 then - print $data00 - return -1 +if $data00 != sma_index_name1 then + if $data00 != sma_index_name2 then + print $data00 + return -1 + endi endi print =============== select * from ct1 from memory @@ -117,7 +136,76 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start sleep 50 +print =============== select * from ct1 from memory after reboot +sql select * from ct1; +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print $data30 $data31 $data32 $data33 +print $data40 $data41 $data42 $data43 +if $rows != 5 then + print rows $rows != 5 + return -1 +endi +print =============== select * from stb from memory in designated vgroup after reboot +sql select _wstart, _wend, min(c1),max(c2),max(c1) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +print =============== select * from stb from memory in common vgroups after reboot +sql select _wstart, _wend, min(c1),max(c2),max(c1),max(c3) from stb interval(5m,10s) sliding(5m); +print $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data02 != -13 then + print data02 $data02 != -13 + return -1 +endi + +if $data03 != 20.00000 then + print data03 $data03 != 20.00000 + return -1 +endi + +if $data04 != 20 then + print data04 $data04 != 20 + return -1 +endi + +if $data05 != 30.000000000 then + print data05 $data05 != 30.000000000 + return -1 +endi + + +#==================== flush database to trigger commit data to file +sql flush database d1; +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sleep 50 print =============== select * from ct1 from file sql select * from ct1; print $data00 $data01 $data02 $data03 diff --git a/tests/script/tsim/stream/checkStreamSTable.sim b/tests/script/tsim/stream/checkStreamSTable.sim new file mode 100644 index 0000000000000000000000000000000000000000..288dd35cfedb6b63d7b97af59ee52d427e2e0353 --- /dev/null +++ b/tests/script/tsim/stream/checkStreamSTable.sim @@ -0,0 +1,500 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result.streamt0(ts timestamp,a int,b int) tags(ta int,tb varchar(100),tc int); + +sql create stream streams0 trigger at_once into result.streamt0 tags(tb) as select _wstart, count(*) c1, max(a) c2 from st partition by tbname tb interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02 +print $data10, $data11, $data12 +print $data20, $data21, $data22 + +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt0 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02 + print $data10, $data11, $data12 + print $data20, $data21, $data22 + goto loop0 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop0 +endi + +if $data03 != NULL then + print =====data03=$data03 + goto loop0 +endi + +if $data04 != t1 then + print =====data04=$data04 + goto loop0 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop0 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop0 +endi + +if $data13 != NULL then + print =====data13=$data13 + goto loop0 +endi + +if $data14 != t2 then + print =====data14=$data14 + goto loop0 +endi + +print ===== step3 + +sql create database result1 vgroups 1; + +sql create database test1 vgroups 4; +sql use test1; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result1.streamt1(ts timestamp,a int,b int,c int) tags(ta varchar(100),tb int,tc int); + +sql create stream streams1 trigger at_once into result1.streamt1(ts,c,a,b) tags(ta) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname as ta interval(10s); +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st partition by tbname interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result1.streamt1 order by ta; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop1 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop1 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop1 +endi + +if $data11 != 40 then + print =====data11=$data11 + goto loop1 +endi + +if $data12 != 50 then + print =====data12=$data12 + goto loop1 +endi + +if $data13 != 1 then + print =====data13=$data13 + goto loop1 +endi + + +print ===== step4 + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stable result2.streamt2(ts timestamp, a int , b int) tags(ta varchar(20)); + +# tag dest 1, source 2 +sql_error create stream streams2 trigger at_once into result2.streamt2 TAGS(aa varchar(100), ta int) as select _wstart, count(*) c1, max(a) from st partition by tbname as aa, ta interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 4 +sql_error create stream streams2 trigger at_once into result2.streamt2(ts, a, b) as select _wstart, count(*) c1, max(a), max(b) from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql_error create stream streams2 trigger at_once into result2.streamt2 as select _wstart, count(*) c1 from st partition by tbname interval(10s); + +# column dest 3, source 2 +sql create stream streams2 trigger at_once into result2.streamt2(ts, a) tags(ta) as select _wstart, count(*) c1 from st partition by tbname as ta interval(10s); + + +print ===== step5 + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result3.streamt3(ts timestamp,a int,b int,c int, d int) tags(ta int,tb int,tc int); + +sql create stream streams3 trigger at_once into result3.streamt3(ts,c,a,b) as select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st interval(10s); +print $data00, $data01, $data02, $data03, $data04 +print $data10, $data11, $data12, $data13, $data14 +print $data20, $data21, $data22, $data23, $data24 + +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 1 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop2 +endi + +if $data01 != 40 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop2 +endi + +if $data03 != 2 then + print =====data03=$data03 + goto loop2 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop2 +endi + +print ===== drop ... + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop database if exists test; +sql drop database if exists test1; +sql drop database if exists test2; +sql drop database if exists test3; +sql drop database if exists result; +sql drop database if exists result1; +sql drop database if exists result2; +sql drop database if exists result3; + +print ===== step6 + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result4.streamt4(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); + +sql create stream streams4 trigger at_once into result4.streamt4(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg1 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); + +sql insert into t1 values(1648791213000,10,20,30); +sql insert into t2 values(1648791213000,40,50,60); + +$loop_count = 0 + +sql select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, tc as tg3 interval(10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by tg1; + +if $rows != 2 then + print =====rows=$rows + print $data00, $data01, $data02, $data03 + print $data10, $data11, $data12, $data13 + print $data20, $data21, $data22, $data23 + goto loop3 +endi + +if $data01 != 10 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 20 then + print =====data02=$data02 + goto loop3 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop3 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop3 +endi + +if $data11 != 40 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 50 then + print =====data12=$data12 + goto loop3 +endi + +if $data13 != 1 then + print =====data13=$data13 + goto loop3 +endi + +if $data14 != NULL then + print =====data14=$data14 + goto loop3 +endi + +print ===== step7 + +sql create database result5 vgroups 1; + +sql create database test5 vgroups 4; +sql use test5; + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,2,3); +sql create table t2 using st tags(4,5,6); + +sql create stable result5.streamt5(ts timestamp,a int,b int,c int, d int) tags(tg1 int,tg2 int,tg3 int); + +sql create stream streams5 trigger at_once into result5.streamt5(ts,c,a,b) tags(tg2, tg3, tg1) subtable( concat("tbl-", cast(tg3 as varchar(10)) ) ) as select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL); + +$loop_count = 0 + +print select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s); +sql select _wstart, count(*) c1, max(a),min(b) c2 from st partition by ta+1 as tg1, cast(tb as bigint) as tg2, a as tg3 session(ts, 10s); +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print sql select * from result5.streamt5 order by tg1; +sql select * from result5.streamt5 order by tg1; +print $data00, $data01, $data02, $data03 $data04 $data05 $data06 $data07 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +if $rows != 1 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != NULL then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop4 +endi + +if $data03 != 1 then + print =====data03=$data03 + goto loop4 +endi + +if $data04 != NULL then + print =====data04=$data04 + goto loop4 +endi + +if $data05 != 2 then + print =====data05=$data05 + goto loop4 +endi + +if $data06 != 2 then + print =====data06=$data06 + goto loop4 +endi + +if $data07 != NULL then + print =====data07=$data07 + goto loop4 +endi + +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop database if exists test4; +sql drop database if exists test5; +sql drop database if exists result4; +sql drop database if exists result5; + +print ===== step8 + +sql drop stream if exists streams8; +sql drop database if exists test8; +sql create database test8 vgroups 1; +sql use test8; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, count(*) c1, count(d) c2, count(c) c3 from t1 partition by tbname interval(10s) ; + +sql drop stream streams8; +sql create stream streams71 trigger at_once into streamt8(ts, c2) tags(group_id)as select _wstart, count(*) from t1 partition by tbname as group_id interval(10s); + +sql insert into t1 values(1648791233000,1,2,3,1.0); + +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt8; +print $data00, $data01, $data02, $data03 +print $data10, $data11, $data12, $data13 +print $data20, $data21, $data22, $data23 + +if $rows != 1 then + print =====rows=$rows + goto loop8 +endi + +if $data01 != NULL then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop8 +endi + +if $data03 != NULL then + print =====data03=$data03 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/fillIntervalDelete0.sim b/tests/script/tsim/stream/fillIntervalDelete0.sim index 77d09d5ae891010dbc154d3aa7b807bc31f86d69..1c0647d57b29a044ce8ae3fb4052423768a0d66e 100644 --- a/tests/script/tsim/stream/fillIntervalDelete0.sim +++ b/tests/script/tsim/stream/fillIntervalDelete0.sim @@ -163,7 +163,7 @@ endi if $data01 != 4 then print =====data01=$data01 - return -1 + goto loop3 endi @@ -239,7 +239,7 @@ endi if $data01 != 6 then print =====data01=$data01 - return -1 + goto loop5 endi diff --git a/tests/script/tsim/stream/fillIntervalDelete1.sim b/tests/script/tsim/stream/fillIntervalDelete1.sim index ca347f120a8dacbc5d6e3d09099cc4e21fc8ed76..0206b88fdca9d97bd8e2bab6c1812c84f44b98f2 100644 --- a/tests/script/tsim/stream/fillIntervalDelete1.sim +++ b/tests/script/tsim/stream/fillIntervalDelete1.sim @@ -325,19 +325,19 @@ if $rows != 12 then endi if $data21 != 3 then print ====streamt10=data21=$data21 - return -1 + goto loop6 endi if $data31 != 4 then print ====streamt10=data31=$data31 - return -1 + goto loop6 endi if $data71 != 8 then print ====streamt10=data71=$data71 - return -1 + goto loop6 endi if $data91 != 10 then print ====streamt10=data91=$data91 - return -1 + goto loop6 endi diff --git a/tests/script/tsim/stream/fillIntervalLinear.sim b/tests/script/tsim/stream/fillIntervalLinear.sim index 4d0f101b6c10f4ca14499c7d59c68ea46a16b8d6..b9f301d81bed7f8fc71a9c4522290c833d46d6ee 100644 --- a/tests/script/tsim/stream/fillIntervalLinear.sim +++ b/tests/script/tsim/stream/fillIntervalLinear.sim @@ -9,6 +9,8 @@ system sh/exec.sh -n dnode1 -s start sleep 200 sql connect +print step 1 start + sql drop stream if exists streams1; sql drop database if exists test1; sql create database test1 vgroups 1; @@ -38,161 +40,165 @@ endi if $data01 != 2.000000000 then print =====data01=$data01 - return -1 + goto loop1 endi if $data02 != 1.000000000 then print =====data02=$data02 - return -1 + goto loop1 endi if $data03 != ccc then print =====data03=$data03 - return -1 + goto loop1 endi if $data04 != 1 then print =====data04=$data04 - return -1 + goto loop1 endi if $data11 != 4.000000000 then print =====data11=$data11 - return -1 + goto loop1 endi + if $data12 != 2.000000000 then print =====data12=$data12 - return -1 + goto loop1 endi + if $data13 != NULL then print =====data13=$data13 - return -1 + goto loop1 endi if $data21 != 6.000000000 then print =====data21=$data21 - return -1 + goto loop1 endi if $data22 != 3.000000000 then print =====data22=$data22 - return -1 + goto loop1 endi if $data23 != NULL then print =====data23=$data23 - return -1 + goto loop1 endi if $data31 != 8.000000000 then print =====data31=$data31 - return -1 + goto loop1 endi if $data32 != 4.000000000 then print =====data32=$data32 - return -1 + goto loop1 endi if $data33 != aaa then print =====data33=$data33 - return -1 + goto loop1 endi if $data41 != 8.666666667 then print =====data41=$data41 - return -1 + goto loop1 endi if $data42 != 4.333333333 then print =====data42=$data42 - return -1 + goto loop1 endi if $data43 != NULL then print =====data43=$data43 - return -1 + goto loop1 endi if $data51 != 9.333333333 then print =====data01=$data01 - return -1 + goto loop1 endi if $data52 != 4.666666667 then print =====data52=$data52 - return -1 + goto loop1 endi if $data53 != NULL then print =====data53=$data53 - return -1 + goto loop1 endi if $data61 != 10.000000000 then print =====data61=$data61 - return -1 + goto loop1 endi if $data62 != 5.000000000 then print =====data62=$data62 - return -1 + goto loop1 endi if $data71 != 8.000000000 then print =====data71=$data71 - return -1 + goto loop1 endi if $data72 != 4.000000000 then print =====data72=$data72 - return -1 + goto loop1 endi if $data81 != 6.000000000 then print =====data81=$data81 - return -1 + goto loop1 endi if $data82 != 3.000000000 then print =====data82=$data82 - return -1 + goto loop1 endi if $data91 != 4.000000000 then print =====data91=$data91 - return -1 + goto loop1 endi if $data92 != 2.000000000 then print =====data92=$data92 - return -1 + goto loop1 endi if $data[10][1] != 4.666666667 then print =====data[10][1]=$data[10][1] - return -1 + goto loop1 endi if $data[10][2] != 2.333333333 then print =====data[10][2]=$data[10][2] - return -1 + goto loop1 endi if $data[11][1] != 5.333333333 then print =====data[11][1]=$data[11][1] - return -1 + goto loop1 endi if $data[11][2] != 2.666666667 then print =====data[11][2]=$data[11][2] - return -1 + goto loop1 endi if $data[12][1] != 6.000000000 then print =====data[12][1]=$data[12][1] - return -1 + goto loop1 endi if $data[12][2] != 3.000000000 then print =====data[12][2]=$data[12][2] - return -1 + goto loop1 endi +print step 1 end +print step 2 start sql drop stream if exists streams2; sql drop database if exists test2; @@ -224,161 +230,163 @@ endi if $data01 != 2.000000000 then print =====data01=$data01 - return -1 + goto loop2 endi if $data02 != 1.000000000 then print =====data02=$data02 - return -1 + goto loop2 endi if $data03 != ccc then print =====data03=$data03 - return -1 + goto loop2 endi if $data04 != 1 then print =====data04=$data04 - return -1 + goto loop2 endi if $data11 != 4.000000000 then print =====data11=$data11 - return -1 + goto loop2 endi if $data12 != 2.000000000 then print =====data12=$data12 - return -1 + goto loop2 endi if $data13 != NULL then print =====data13=$data13 - return -1 + goto loop2 endi if $data21 != 6.000000000 then print =====data21=$data21 - return -1 + goto loop2 endi if $data22 != 3.000000000 then print =====data22=$data22 - return -1 + goto loop2 endi if $data23 != NULL then print =====data23=$data23 - return -1 + goto loop2 endi if $data31 != 8.000000000 then print =====data31=$data31 - return -1 + goto loop2 endi if $data32 != 4.000000000 then print =====data32=$data32 - return -1 + goto loop2 endi if $data33 != aaa then print =====data33=$data33 - return -1 + goto loop2 endi if $data41 != 8.666666667 then print =====data41=$data41 - return -1 + goto loop2 endi if $data42 != 4.333333333 then print =====data42=$data42 - return -1 + goto loop2 endi if $data43 != NULL then print =====data43=$data43 - return -1 + goto loop2 endi if $data51 != 9.333333333 then print =====data01=$data01 - return -1 + goto loop2 endi if $data52 != 4.666666667 then print =====data52=$data52 - return -1 + goto loop2 endi if $data53 != NULL then print =====data53=$data53 - return -1 + goto loop2 endi if $data61 != 10.000000000 then print =====data61=$data61 - return -1 + goto loop2 endi if $data62 != 5.000000000 then print =====data62=$data62 - return -1 + goto loop2 endi if $data71 != 8.000000000 then print =====data71=$data71 - return -1 + goto loop2 endi if $data72 != 4.000000000 then print =====data72=$data72 - return -1 + goto loop2 endi if $data81 != 6.000000000 then print =====data81=$data81 - return -1 + goto loop2 endi if $data82 != 3.000000000 then print =====data82=$data82 - return -1 + goto loop2 endi if $data91 != 4.000000000 then print =====data91=$data91 - return -1 + goto loop2 endi if $data92 != 2.000000000 then print =====data92=$data92 - return -1 + goto loop2 endi if $data[10][1] != 4.666666667 then print =====data[10][1]=$data[10][1] - return -1 + goto loop2 endi if $data[10][2] != 2.333333333 then print =====data[10][2]=$data[10][2] - return -1 + goto loop2 endi if $data[11][1] != 5.333333333 then print =====data[11][1]=$data[11][1] - return -1 + goto loop2 endi if $data[11][2] != 2.666666667 then print =====data[11][2]=$data[11][2] - return -1 + goto loop2 endi if $data[12][1] != 6.000000000 then print =====data[12][1]=$data[12][1] - return -1 + goto loop2 endi if $data[12][2] != 3.000000000 then print =====data[12][2]=$data[12][2] - return -1 + goto loop2 endi +print step 2 end +print step 3 start sql drop stream if exists streams3; sql drop database if exists test3; @@ -412,98 +420,98 @@ endi if $data01 != 3 then print =====data01=$data01 - return -1 + goto loop3 endi if $data02 != 6.000000000 then print =====data02=$data02 - return -1 + goto loop3 endi if $data03 != ccc then print =====data03=$data03 - return -1 + goto loop3 endi if $data11 != 3 then print =====data11=$data11 - return -1 + goto loop3 endi if $data12 != 7.000000000 then print =====data12=$data12 - return -1 + goto loop3 endi if $data13 != NULL then print =====data13=$data13 - return -1 + goto loop3 endi if $data21 != 4 then print =====data21=$data21 - return -1 + goto loop3 endi if $data22 != 8.000000000 then print =====data22=$data22 - return -1 + goto loop3 endi if $data23 != ddd then print =====data23=$data23 - return -1 + goto loop3 endi if $data31 != 2 then print =====data31=$data31 - return -1 + goto loop3 endi if $data32 != 5.000000000 then print =====data32=$data32 - return -1 + goto loop3 endi if $data33 != NULL then print =====data33=$data33 - return -1 + goto loop3 endi if $data41 != 1 then print =====data41=$data41 - return -1 + goto loop3 endi if $data42 != 2.000000000 then print =====data42=$data42 - return -1 + goto loop3 endi if $data43 != aaa then print =====data43=$data43 - return -1 + goto loop3 endi if $data51 != 1 then print =====data51=$data51 - return -1 + goto loop3 endi if $data52 != 3.000000000 then print =====data52=$data52 - return -1 + goto loop3 endi if $data53 != NULL then print =====data53=$data53 - return -1 + goto loop3 endi if $data61 != 2 then print =====data61=$data61 - return -1 + goto loop3 endi if $data62 != 4.000000000 then print =====data62=$data62 - return -1 + goto loop3 endi if $data63 != bbb then print =====data63=$data63 - return -1 + goto loop3 endi @@ -534,42 +542,42 @@ endi if $data01 != 6 then print =====data01=$data01 - return -1 + goto loop4 endi if $data02 != 12.000000000 then print =====data02=$data02 - return -1 + goto loop4 endi if $data03 != fff then print =====data03=$data03 - return -1 + goto loop4 endi if $data11 != 6 then print =====data11=$data11 - return -1 + goto loop4 endi if $data12 != 13.000000000 then print =====data12=$data12 - return -1 + goto loop4 endi if $data13 != NULL then print =====data13=$data13 - return -1 + goto loop4 endi if $data21 != 7 then print =====data21=$data21 - return -1 + goto loop4 endi if $data22 != 14.000000000 then print =====data22=$data22 - return -1 + goto loop4 endi if $data23 != ggg then print =====data23=$data23 - return -1 + goto loop4 endi @@ -626,33 +634,10 @@ if $data[12][3] != hhh then return -1 endi -if $data[13][1] != 8 then - print =====data[13][1]=$data[13][1] - return -1 -endi -if $data[13][2] != 17.000000000 then - print =====data[13][2]=$data[13][2] - return -1 -endi -if $data[13][3] != NULL then - print =====data[13][3]=$data[13][3] - return -1 -endi -if $data[14][1] != 9 then - print =====data[14][1]=$data[14][1] - return -1 -endi -if $data[14][2] != 18.000000000 then - print =====data[14][2]=$data[14][2] - return -1 -endi -if $data[14][3] != iii then - print =====data[14][3]=$data[14][3] - return -1 -endi +print step 3 end diff --git a/tests/script/tsim/stream/fillIntervalPartitionBy.sim b/tests/script/tsim/stream/fillIntervalPartitionBy.sim index 8c06ca8bbdfe5ffd99ea81513a56827f5f22376d..168452fdc8950223a944b1b18e23e0a3dd508873 100644 --- a/tests/script/tsim/stream/fillIntervalPartitionBy.sim +++ b/tests/script/tsim/stream/fillIntervalPartitionBy.sim @@ -149,7 +149,7 @@ endi if $data04 == 0 then print ====streamt1=data04=$data04 - return -1 + goto loop4 endi sql select group_id,count(*) from streamt1 group by group_id; @@ -168,7 +168,7 @@ endi if $data04 == 0 then print ====streamt2=data04=$data04 - return -1 + goto loop4 endi sql select group_id,count(*) from streamt2 group by group_id; @@ -187,7 +187,7 @@ endi if $data04 == 0 then print ====streamt3=data04=$data04 - return -1 + goto loop4 endi sql select group_id,count(*) from streamt3 group by group_id; @@ -207,7 +207,7 @@ endi if $data04 == 0 then print ====streamt4=data04=$data04 - return -1 + goto loop4 endi sql select group_id,count(*) from streamt4 group by group_id; @@ -226,7 +226,7 @@ endi if $data04 == 0 then print ====streamt5=data04=$data04 - return -1 + goto loop4 endi sql select group_id,count(*) from streamt5 group by group_id; diff --git a/tests/script/tsim/stream/fillIntervalPrevNext.sim b/tests/script/tsim/stream/fillIntervalPrevNext.sim index 4eadd7e7b1b976bcd7b78b5a4a3ec37de6a5549d..99f7dcb5cb03aa9734efae6bfe3429c98c7d9ad8 100644 --- a/tests/script/tsim/stream/fillIntervalPrevNext.sim +++ b/tests/script/tsim/stream/fillIntervalPrevNext.sim @@ -228,31 +228,31 @@ endi if $data72 != 6.000000000 then print =====data72=$data72 - return -1 + goto loop1 endi if $data73 != 3.000000000 then print =====data73=$data73 - return -1 + goto loop1 endi if $data82 != 6.000000000 then print =====data82=$data82 - return -1 + goto loop1 endi if $data83 != 3.000000000 then print =====data83=$data83 - return -1 + goto loop1 endi if $data92 != 6.000000000 then print =====data92=$data92 - return -1 + goto loop1 endi if $data93 != 3.000000000 then print =====data93=$data93 - return -1 + goto loop1 endi @@ -483,535 +483,33 @@ endi if $data72 != 6.000000000 then print =====data72=$data72 - return -1 + goto loop6 endi if $data73 != 3.000000000 then print =====data73=$data73 - return -1 + goto loop6 endi if $data82 != 6.000000000 then print =====data82=$data82 - return -1 + goto loop6 endi if $data83 != 3.000000000 then print =====data83=$data83 - return -1 + goto loop6 endi if $data92 != 6.000000000 then print =====data92=$data92 - return -1 + goto loop6 endi if $data93 != 3.000000000 then print =====data93=$data93 - return -1 -endi - - - -sql drop stream if exists streams7; -sql drop stream if exists streams8; -sql drop database if exists test7; -sql create database test7 vgroups 1; -sql use test7; -sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); -sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); -sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); -sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); -sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); -sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); -sql insert into t1 values(1648791213000,4,4,4,4.0,'ddd'); - - -$loop_count = 0 - -loop7: -sleep 300 -sql select * from streamt7 order by ts; - -$loop_count = $loop_count + 1 -if $loop_count == 10 then - return -1 -endi - - -if $rows != 7 then - print =====rows=$rows - goto loop7 -endi - - -if $data01 != 3 then - print =====data01=$data01 - return -1 -endi -if $data02 != 6.000000000 then - print =====data02=$data02 - return -1 -endi -if $data03 != ccc then - print =====data03=$data03 - return -1 -endi - -if $data11 != 3 then - print =====data11=$data11 - return -1 -endi -if $data12 != 6.000000000 then - print =====data12=$data12 - return -1 -endi -if $data13 != ccc then - print =====data13=$data13 - return -1 -endi - - -if $data21 != 4 then - print =====data21=$data21 - return -1 -endi -if $data22 != 8.000000000 then - print =====data22=$data22 - return -1 -endi -if $data23 != ddd then - print =====data23=$data23 - return -1 -endi - - -if $data31 != 4 then - print =====data31=$data31 - return -1 -endi -if $data32 != 8.000000000 then - print =====data32=$data32 - return -1 -endi -if $data33 != ddd then - print =====data33=$data33 - return -1 -endi - - -if $data41 != 1 then - print =====data41=$data41 - return -1 -endi -if $data42 != 2.000000000 then - print =====data42=$data42 - return -1 -endi -if $data43 != aaa then - print =====data43=$data43 - return -1 -endi - - -if $data51 != 1 then - print =====data51=$data51 - return -1 -endi -if $data52 != 2.000000000 then - print =====data52=$data52 - return -1 -endi -if $data53 != aaa then - print =====data53=$data53 - return -1 -endi - - -if $data61 != 2 then - print =====data61=$data61 - return -1 -endi -if $data62 != 4.000000000 then - print =====data62=$data62 - return -1 -endi -if $data63 != bbb then - print =====data63=$data63 - return -1 -endi - -#-------------- - -sleep 200 -sql select * from streamt8 order by ts; - - -if $rows != 7 then - print =====rows=$rows - return -1 -endi - - -if $data01 != 3 then - print =====data01=$data01 - return -1 -endi -if $data02 != 1 then - print =====data02=$data02 - return -1 -endi -if $data03 != 4.000000000 then - print =====data03=$data03 - return -1 -endi - -if $data11 != 4 then - print =====data11=$data11 - return -1 -endi -if $data12 != 1 then - print =====data12=$data12 - return -1 -endi -if $data13 != 5.000000000 then - print =====data13=$data13 - return -1 -endi - - -if $data21 != 4 then - print =====data21=$data21 - return -1 -endi -if $data22 != 1 then - print =====data22=$data22 - return -1 -endi -if $data23 != 5.000000000 then - print =====data23=$data23 - return -1 -endi - - -if $data31 != 1 then - print =====data31=$data31 - return -1 -endi -if $data32 != 1 then - print =====data32=$data32 - return -1 -endi -if $data33 != 2.000000000 then - print =====data33=$data33 - return -1 -endi - - -if $data41 != 1 then - print =====data41=$data41 - return -1 -endi -if $data42 != 1 then - print =====data42=$data42 - return -1 -endi -if $data43 != 2.000000000 then - print =====data43=$data43 - return -1 -endi - - -if $data51 != 2 then - print =====data51=$data51 - return -1 -endi -if $data52 != 1 then - print =====data52=$data52 - return -1 -endi -if $data53 != 3.000000000 then - print =====data53=$data53 - return -1 -endi - - -if $data61 != 2 then - print =====data61=$data61 - return -1 -endi -if $data62 != 1 then - print =====data62=$data62 - return -1 -endi -if $data63 != 3.000000000 then - print =====data63=$data63 - return -1 -endi - -sql insert into t1 values(1648791212000,5,5,5,5.0,'eee'); -sql insert into t1 values(1648791207000,6,6,6,6.0,'fff') (1648791209000,7,7,7,7.0,'ggg') (1648791219000,8,8,8,8.0,'hhh') (1648791221000,9,9,9,9.0,'iii'); - - - -$loop_count = 0 - -loop8: -sleep 200 -sql select * from streamt7 order by ts; - -$loop_count = $loop_count + 1 -if $loop_count == 10 then - return -1 -endi - - -if $rows != 15 then - print =====rows=$rows - goto loop8 -endi - - -if $data01 != 6 then - print =====data01=$data01 - return -1 -endi -if $data02 != 12.000000000 then - print =====data02=$data02 - return -1 -endi -if $data03 != fff then - print =====data03=$data03 - return -1 -endi - -if $data11 != 6 then - print =====data11=$data11 - return -1 -endi -if $data12 != 12.000000000 then - print =====data12=$data12 - return -1 -endi -if $data13 != fff then - print =====data13=$data13 - return -1 -endi - - -if $data21 != 7 then - print =====data21=$data21 - return -1 -endi -if $data22 != 14.000000000 then - print =====data22=$data22 - return -1 -endi -if $data23 != ggg then - print =====data23=$data23 - return -1 -endi - - -if $data31 != 7 then - print =====data31=$data31 - return -1 -endi -if $data32 != 14.000000000 then - print =====data32=$data32 - return -1 -endi -if $data33 != ggg then - print =====data33=$data33 - return -1 -endi - -if $data51 != 5 then - print =====data51=$data51 - return -1 -endi -if $data52 != 10.000000000 then - print =====data52=$data52 - return -1 -endi -if $data53 != eee then - print =====data53=$data53 - return -1 -endi - - -if $data[11][1] != 2 then - print =====data[11][1]=$data[11][1] - return -1 -endi -if $data[11][2] != 4.000000000 then - print =====data[11][2]=$data[11][2] - return -1 -endi -if $data[11][3] != bbb then - print =====data[11][3]=$data[11][3] - return -1 -endi - -if $data[12][1] != 8 then - print =====data[12][1]=$data[12][1] - return -1 -endi -if $data[12][2] != 16.000000000 then - print =====data[12][2]=$data[12][2] - return -1 -endi -if $data[12][3] != hhh then - print =====data[12][3]=$data[12][3] - return -1 -endi - -if $data[13][1] != 8 then - print =====data[13][1]=$data[13][1] - return -1 -endi -if $data[13][2] != 16.000000000 then - print =====data[13][2]=$data[13][2] - return -1 -endi -if $data[13][3] != hhh then - print =====data[13][3]=$data[13][3] - return -1 -endi - -if $data[14][1] != 9 then - print =====data[14][1]=$data[14][1] - return -1 -endi -if $data[14][2] != 18.000000000 then - print =====data[14][2]=$data[14][2] - return -1 -endi -if $data[14][3] != iii then - print =====data[14][3]=$data[14][3] - return -1 -endi - -print fill next-----------------890 -sql use test7; -sql select * from streamt8 order by ts; - -if $rows != 15 then - print =====rows=$rows - goto loop8 -endi - - -if $data01 != 6 then - print =====data01=$data01 - return -1 -endi -if $data02 != 1 then - print =====data02=$data02 - return -1 -endi -if $data03 != 7.000000000 then - print =====data03=$data03 - return -1 -endi - -if $data11 != 7 then - print =====data11=$data11 - return -1 -endi -if $data13 != 8.000000000 then - print =====data13=$data13 - return -1 -endi - - -if $data21 != 7 then - print =====data21=$data21 - return -1 -endi -if $data23 != 8.000000000 then - print =====data23=$data23 - return -1 -endi - - -if $data31 != 3 then - print =====data31=$data31 - return -1 -endi -if $data33 != 4.000000000 then - print =====data33=$data33 - return -1 -endi - -if $data51 != 5 then - print =====data51=$data51 - return -1 -endi -if $data53 != 6.000000000 then - print =====data53=$data53 - return -1 -endi - - -if $data[11][1] != 8 then - print =====data[11][1]=$data[11][1] - return -1 -endi -if $data[11][2] != 1 then - print =====data[11][2]=$data[11][2] - return -1 -endi -if $data[11][3] != 9.000000000 then - print =====data[11][3]=$data[11][3] - return -1 -endi - -if $data[12][1] != 8 then - print =====data[12][1]=$data[12][1] - return -1 -endi -if $data[12][3] != 9.000000000 then - print =====data[12][3]=$data[12][3] - return -1 -endi - -if $data[13][1] != 9 then - print =====data[13][1]=$data[13][1] - return -1 -endi -if $data[13][3] != 10.000000000 then - print =====data[13][3]=$data[13][3] - return -1 -endi - -if $data[14][1] != 9 then - print =====data[14][1]=$data[14][1] - return -1 -endi -if $data[14][3] != 10.000000000 then - print =====data[14][3]=$data[14][3] - return -1 + goto loop6 endi - - - - - - - - - - - - - - - - - - - #==system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/fillIntervalPrevNext1.sim b/tests/script/tsim/stream/fillIntervalPrevNext1.sim new file mode 100644 index 0000000000000000000000000000000000000000..8058065bcf7d22938ea49221919e794e7a58e933 --- /dev/null +++ b/tests/script/tsim/stream/fillIntervalPrevNext1.sim @@ -0,0 +1,559 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +#==system sh/exec.sh -n dnode1 -s start -v + +sleep 200 +sql connect + + +sql drop stream if exists streams7; +sql drop stream if exists streams8; +sql drop database if exists test7; +sql create database test7 vgroups 1; +sql use test7; +sql create table t1(ts timestamp, a int, b int , c int, d double, s varchar(20)); +sql create stream streams7 trigger at_once into streamt7 as select _wstart as ts, max(a), b+c, s from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(prev); +sql create stream streams8 trigger at_once into streamt8 as select _wstart as ts, max(a), 1, b+1 from t1 where ts >= 1648791150000 and ts < 1648791261000 interval(1s) fill(next); +sql insert into t1 values(1648791215000,1,1,1,1.0,'aaa'); +sql insert into t1 values(1648791217000,2,2,2,2.0,'bbb'); +sql insert into t1 values(1648791211000,3,3,3,3.0,'ccc'); +sql insert into t1 values(1648791213000,4,4,4,4.0,'ddd'); + + +$loop_count = 0 + +loop7: +sleep 300 +sql select * from streamt7 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 7 then + print =====rows=$rows + goto loop7 +endi + + +if $data01 != 3 then + print =====data01=$data01 + goto loop7 +endi +if $data02 != 6.000000000 then + print =====data02=$data02 + goto loop7 +endi +if $data03 != ccc then + print =====data03=$data03 + goto loop7 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop7 +endi +if $data12 != 6.000000000 then + print =====data12=$data12 + goto loop7 +endi +if $data13 != ccc then + print =====data13=$data13 + goto loop7 +endi + + +if $data21 != 4 then + print =====data21=$data21 + goto loop7 +endi +if $data22 != 8.000000000 then + print =====data22=$data22 + goto loop7 +endi +if $data23 != ddd then + print =====data23=$data23 + goto loop7 +endi + + +if $data31 != 4 then + print =====data31=$data31 + goto loop7 +endi +if $data32 != 8.000000000 then + print =====data32=$data32 + goto loop7 +endi +if $data33 != ddd then + print =====data33=$data33 + goto loop7 +endi + + +if $data41 != 1 then + print =====data41=$data41 + goto loop7 +endi +if $data42 != 2.000000000 then + print =====data42=$data42 + goto loop7 +endi +if $data43 != aaa then + print =====data43=$data43 + goto loop7 +endi + + +if $data51 != 1 then + print =====data51=$data51 + goto loop7 +endi +if $data52 != 2.000000000 then + print =====data52=$data52 + goto loop7 +endi +if $data53 != aaa then + print =====data53=$data53 + goto loop7 +endi + + +if $data61 != 2 then + print =====data61=$data61 + goto loop7 +endi +if $data62 != 4.000000000 then + print =====data62=$data62 + goto loop7 +endi +if $data63 != bbb then + print =====data63=$data63 + goto loop7 +endi + +#-------------- + +loop71: + +$loop_count = 0 +sleep 300 + +sql select * from streamt8 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 7 then + print =====rows=$rows + goto loop71 +endi + + +if $data01 != 3 then + print =====data01=$data01 + goto loop71 +endi +if $data02 != 1 then + print =====data02=$data02 + goto loop71 +endi +if $data03 != 4.000000000 then + print =====data03=$data03 + goto loop71 +endi + +if $data11 != 4 then + print =====data11=$data11 + goto loop71 +endi +if $data12 != 1 then + print =====data12=$data12 + goto loop71 +endi +if $data13 != 5.000000000 then + print =====data13=$data13 + goto loop71 +endi + + +if $data21 != 4 then + print =====data21=$data21 + goto loop71 +endi +if $data22 != 1 then + print =====data22=$data22 + goto loop71 +endi +if $data23 != 5.000000000 then + print =====data23=$data23 + goto loop71 +endi + + +if $data31 != 1 then + print =====data31=$data31 + goto loop71 +endi +if $data32 != 1 then + print =====data32=$data32 + goto loop71 +endi +if $data33 != 2.000000000 then + print =====data33=$data33 + goto loop71 +endi + + +if $data41 != 1 then + print =====data41=$data41 + goto loop71 +endi +if $data42 != 1 then + print =====data42=$data42 + goto loop71 +endi +if $data43 != 2.000000000 then + print =====data43=$data43 + goto loop71 +endi + + +if $data51 != 2 then + print =====data51=$data51 + goto loop71 +endi +if $data52 != 1 then + print =====data52=$data52 + goto loop71 +endi +if $data53 != 3.000000000 then + print =====data53=$data53 + goto loop71 +endi + + +if $data61 != 2 then + print =====data61=$data61 + goto loop71 +endi +if $data62 != 1 then + print =====data62=$data62 + goto loop71 +endi +if $data63 != 3.000000000 then + print =====data63=$data63 + goto loop71 +endi + +sql insert into t1 values(1648791212000,5,5,5,5.0,'eee'); +sql insert into t1 values(1648791207000,6,6,6,6.0,'fff') (1648791209000,7,7,7,7.0,'ggg') (1648791219000,8,8,8,8.0,'hhh') (1648791221000,9,9,9,9.0,'iii'); + + + +$loop_count = 0 + +loop8: +sleep 200 +sql select * from streamt7 order by ts; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + + +if $rows != 15 then + print =====rows=$rows + goto loop8 +endi + + +if $data01 != 6 then + print =====data01=$data01 + goto loop8 +endi +if $data02 != 12.000000000 then + print =====data02=$data02 + goto loop8 +endi +if $data03 != fff then + print =====data03=$data03 + goto loop8 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop8 +endi +if $data12 != 12.000000000 then + print =====data12=$data12 + goto loop8 +endi +if $data13 != fff then + print =====data13=$data13 + goto loop8 +endi + + +if $data21 != 7 then + print =====data21=$data21 + goto loop8 +endi +if $data22 != 14.000000000 then + print =====data22=$data22 + goto loop8 +endi +if $data23 != ggg then + print =====data23=$data23 + goto loop8 +endi + + +if $data31 != 7 then + print =====data31=$data31 + goto loop8 +endi +if $data32 != 14.000000000 then + print =====data32=$data32 + goto loop8 +endi +if $data33 != ggg then + print =====data33=$data33 + goto loop8 +endi + +if $data51 != 5 then + print =====data51=$data51 + goto loop8 +endi +if $data52 != 10.000000000 then + print =====data52=$data52 + goto loop8 +endi +if $data53 != eee then + print =====data53=$data53 + goto loop8 +endi + + +if $data[11][1] != 2 then + print =====data[11][1]=$data[11][1] + goto loop8 +endi +if $data[11][2] != 4.000000000 then + print =====data[11][2]=$data[11][2] + goto loop8 +endi +if $data[11][3] != bbb then + print =====data[11][3]=$data[11][3] + goto loop8 +endi + +if $data[12][1] != 8 then + print =====data[12][1]=$data[12][1] + goto loop8 +endi +if $data[12][2] != 16.000000000 then + print =====data[12][2]=$data[12][2] + goto loop8 +endi +if $data[12][3] != hhh then + print =====data[12][3]=$data[12][3] + goto loop8 +endi + +if $data[13][1] != 8 then + print =====data[13][1]=$data[13][1] + goto loop8 +endi +if $data[13][2] != 16.000000000 then + print =====data[13][2]=$data[13][2] + goto loop8 +endi +if $data[13][3] != hhh then + print =====data[13][3]=$data[13][3] + goto loop8 +endi + +if $data[14][1] != 9 then + print =====data[14][1]=$data[14][1] + goto loop8 +endi +if $data[14][2] != 18.000000000 then + print =====data[14][2]=$data[14][2] + goto loop8 +endi +if $data[14][3] != iii then + print =====data[14][3]=$data[14][3] + goto loop8 +endi + +print fill next-----------------890 +sql use test7; +sql select * from streamt8 order by ts; + +if $rows != 15 then + print =====rows=$rows + goto loop8 +endi + + +if $data01 != 6 then + print =====data01=$data01 + goto loop8 +endi +if $data02 != 1 then + print =====data02=$data02 + goto loop8 +endi +if $data03 != 7.000000000 then + print =====data03=$data03 + goto loop8 +endi + +if $data11 != 7 then + print =====data11=$data11 + goto loop8 +endi +if $data13 != 8.000000000 then + print =====data13=$data13 + goto loop8 +endi + + +if $data21 != 7 then + print =====data21=$data21 + goto loop8 +endi +if $data23 != 8.000000000 then + print =====data23=$data23 + goto loop8 +endi + + +if $data31 != 3 then + print =====data31=$data31 + goto loop8 +endi +if $data33 != 4.000000000 then + print =====data33=$data33 + goto loop8 +endi + +if $data51 != 5 then + print =====data51=$data51 + goto loop8 +endi +if $data53 != 6.000000000 then + print =====data53=$data53 + goto loop8 +endi + + +if $data[11][1] != 8 then + print =====data[11][1]=$data[11][1] + goto loop8 +endi +if $data[11][2] != 1 then + print =====data[11][2]=$data[11][2] + goto loop8 +endi +if $data[11][3] != 9.000000000 then + print =====data[11][3]=$data[11][3] + goto loop8 +endi + +if $data[12][1] != 8 then + print =====data[12][1]=$data[12][1] + goto loop8 +endi +if $data[12][3] != 9.000000000 then + print =====data[12][3]=$data[12][3] + goto loop8 +endi + +if $data[13][1] != 9 then + print =====data[13][1]=$data[13][1] + goto loop8 +endi +if $data[13][3] != 10.000000000 then + print =====data[13][3]=$data[13][3] + goto loop8 +endi + +if $data[14][1] != 9 then + print =====data[14][1]=$data[14][1] + goto loop8 +endi +if $data[14][3] != 10.000000000 then + print =====data[14][3]=$data[14][3] + goto loop8 +endi + + + + + + + + + + + + + + + + + + + + + + +#==system sh/exec.sh -n dnode1 -s stop -x SIGINT +#==print =============== check +#==$null= + +#==system_content sh/checkValgrind.sh -n dnode1 +#==print cmd return result ----> [ $system_content ] +#==if $system_content > 0 then +#== return -1 +#==endi + +#==if $system_content == $null then +#== return -1 +#==endi +#==return 1 + + + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; +sql drop stream if exists streams6; +sql drop stream if exists streams7; +sql drop stream if exists streams8; + +sql use test7; +sql select * from t1; +print $data00 + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/ignoreExpiredData.sim b/tests/script/tsim/stream/ignoreExpiredData.sim index b143b7977fe6e1de9e716ba32d153d076830478a..fb6cf2eec6a44d0f25c31e4c3fa79f6e352c69ce 100644 --- a/tests/script/tsim/stream/ignoreExpiredData.sim +++ b/tests/script/tsim/stream/ignoreExpiredData.sim @@ -116,7 +116,7 @@ sql create stream stream_t1 trigger at_once IGNORE EXPIRED 1 into streamtST1 as sql create stream stream_t2 trigger at_once IGNORE EXPIRED 1 into streamtST2 as select _wstart, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st session(ts, 10s) ; sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791222001,2,2,3); -sleep 300 +sleep 200 sql insert into ts2 values(1648791211000,1,2,3); sql insert into ts2 values(1648791222001,2,2,3); @@ -160,4 +160,4 @@ if $data02 != 1 then goto loop5 endi -system sh/stop_dnodes.sh \ No newline at end of file +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/udTableAndTag0.sim b/tests/script/tsim/stream/udTableAndTag0.sim new file mode 100644 index 0000000000000000000000000000000000000000..8bf34dc54c222ab54126564a877d5304bcfcf168 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag0.sim @@ -0,0 +1,446 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE(concat("aaa-", tbname)) as select _wstart, count(*) c1 from st partition by tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 + print $data10 + print $data20 + print $data30 + goto loop0 +endi + +if $data00 != aaa-t1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-t2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== tag name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print data00 != cc + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != tag-t1 then + print data00 != tag-t1 + print =====data00=$data00 + goto loop2 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== tag name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,1,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != tag-t1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != tag-t2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop6 +endi + + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", tbname)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != tag-t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != tag-t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != tag-t3 then + print =====data22=$data22 + goto loop8 +endi + +print ===== step6 +print ===== transform tag value + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop stream if exists streams5; + +sql drop database if exists test1; +sql drop database if exists test2; +sql drop database if exists test3; +sql drop database if exists test4; +sql drop database if exists test5; + +sql drop database if exists result1; +sql drop database if exists result2; +sql drop database if exists result3; +sql drop database if exists result4; +sql drop database if exists result5; + + + +sql create database result6 vgroups 1; + +sql create database test6 vgroups 4; +sql use test6; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta varchar(20), tb int, tc int); +sql create table t1 using st tags("1",1,1); +sql create table t2 using st tags("2",2,2); +sql create table t3 using st tags("3",3,3); + +sql create stream streams6 trigger at_once into result6.streamt6 TAGS(dd int) as select _wstart, count(*) c1 from st partition by concat(ta, "0") as dd, tbname interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop9: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result6.streamt6 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop9 +endi + +if $data02 != 10 then + print =====data02=$data02 + goto loop9 +endi + +if $data12 != 20 then + print =====data12=$data12 + goto loop9 +endi + +if $data22 != 30 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/udTableAndTag1.sim b/tests/script/tsim/stream/udTableAndTag1.sim new file mode 100644 index 0000000000000000000000000000000000000000..4229de2cf0701c47aa12d9a196cb41fe4c39ca31 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag1.sim @@ -0,0 +1,373 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +#sql_error create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams1 trigger at_once into result.streamt SUBTABLE( concat("aaa-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by a interval(10s); +print ===== insert into 1 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + print $data20 $data30 + goto loop0 +endi + +if $data00 != aaa-1 then + print =====data00=$data00 + goto loop0 +endi + +if $data10 != aaa-2 then + print =====data10=$data10 + goto loop0 +endi + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + + +print ===== step3 +print ===== column name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as cc interval(10s); +print ===== insert into 2 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != cc then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != cc then + print =====data10=$data10 + goto loop2 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop2 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop2 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop2 +endi + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop3 +endi + + +print ===== step4 +print ===== column name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", cast(a as varchar(10) ) ) ) as select _wstart, count(*) c1 from st partition by concat("col-", cast(a as varchar(10) ) ) as dd, a interval(10s); +print ===== insert into 3 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != dd then + print =====data10=$data10 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != col-1 then + print =====data00=$data00 + goto loop4 +endi + +if $data10 != col-2 then + print =====data10=$data10 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 2 then + print =====rows=$rows + print $data00 $data10 + goto loop6 +endi + +if $data00 != tbn-1 then + print =====data00=$data00 + goto loop6 +endi + +if $data10 != tbn-2 then + print =====data10=$data10 + goto loop6 +endi + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 4; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", dd)) as select _wstart, count(*) c1 from st partition by concat("t", cast(a as varchar(10) ) ) as dd interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop7 +endi + +if $data00 != tbn-t1 then + print =====data00=$data00 + goto loop7 +endi + +if $data10 != tbn-t2 then + print =====data10=$data10 + goto loop7 +endi + +if $data20 != tbn-t3 then + print =====data20=$data20 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 3 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != t1 then + print =====data02=$data02 + goto loop8 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop8 +endi + +if $data12 != t2 then + print =====data12=$data12 + goto loop8 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop8 +endi + +if $data22 != t3 then + print =====data22=$data22 + goto loop8 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/udTableAndTag2.sim b/tests/script/tsim/stream/udTableAndTag2.sim new file mode 100644 index 0000000000000000000000000000000000000000..bacc301ad005d54dadd098a1b88db9ab17427c16 --- /dev/null +++ b/tests/script/tsim/stream/udTableAndTag2.sim @@ -0,0 +1,515 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ===== step1 + +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print ===== step2 +print ===== table name + +sql create database result vgroups 1; + +sql create database test vgroups 4; +sql use test; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams1 trigger at_once into result.streamt SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +print ===== insert into 1 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 +loop0: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop0 +endi + +if $data00 != aaa then + print =====data00=$data00 + goto loop0 +endi + + +$loop_count = 0 +loop1: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result.streamt; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop1 +endi + +# group id +if $data02 == NULL then + print =====data02=$data02 + goto loop1 +endi + + +print ===== step3 +print ===== column name + +sql create database result2 vgroups 1; + +sql create database test2 vgroups 4; +sql use test2; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams2 trigger at_once into result2.streamt2 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); +print ===== insert into 2 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop2: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +sql select tag_name from information_schema.ins_tags where db_name="result2" and stable_name = "streamt2" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop2 +endi + +if $data00 != cc then + print =====data00=$data00 + goto loop2 +endi + +print sql select cc from result2.streamt2 order by 1; + +$loop_count = 0 +loop21: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select cc from result2.streamt2 order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop21 +endi + +if $data00 != NULL then + print =====data00=$data00 + goto loop21 +endi + + +$loop_count = 0 +loop3: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result2.streamt2; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop3 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop3 +endi + + +print ===== step4 +print ===== column name + table name + +sql create database result3 vgroups 1; + +sql create database test3 vgroups 4; +sql use test3; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once into result3.streamt3 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1") ) as select _wstart, count(*) c1 from st interval(10s); +print ===== insert into 3 +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + + +$loop_count = 0 +loop4: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select tag_name from information_schema.ins_tags where db_name="result3" and stable_name = "streamt3" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop4 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop4 +endi + +sql select dd from result3.streamt3 order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop4 +endi + +if $data00 != NULL then + print =====data00=$data00 + goto loop4 +endi + +$loop_count = 0 +loop5: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result3.streamt3; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop5 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop5 +endi + +$loop_count = 0 +loop6: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result3" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop6 +endi + +if $data00 != tbn-1 then + print =====data00=$data00 + goto loop6 +endi + +print ===== step5 +print ===== tag name + table name + +sql create database result4 vgroups 1; + +sql create database test4 vgroups 1; +sql use test4; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); + +sql create stream streams4 trigger at_once into result4.streamt4 TAGS(dd varchar(100)) SUBTABLE(concat("tbn-", "1")) as select _wstart, count(*) c1 from st interval(10s); +sql insert into t1 values(1648791213000,1,1,1) t2 values(1648791213000,2,2,2) t3 values(1648791213000,3,3,3); + + +$loop_count = 0 +loop7: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select table_name from information_schema.ins_tables where db_name="result4" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop7 +endi + +if $data00 != tbn-1 then + print =====data00=$data00 + goto loop7 +endi + +$loop_count = 0 +loop8: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result4.streamt4 order by 3; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop8 +endi + +if $data01 != 3 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + + +print ===== step6 +print ===== table name + +sql create database result5 vgroups 1; + +sql create database test5 vgroups 1; +sql use test5; + + +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams51 trigger at_once into result5.streamt51 SUBTABLE("aaa") as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams52 trigger at_once into result5.streamt52 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st interval(10s); +sql create stream streams53 trigger at_once into result5.streamt53 TAGS(dd varchar(100)) SUBTABLE(concat("aaa-", "1") ) as select _wstart, count(*) c1 from st interval(10s); + +sql insert into t1 values(1648791213000,1,2,3); +sql insert into t2 values(1648791213000,2,2,3); + +$loop_count = 0 +loop9: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select table_name from information_schema.ins_tables where db_name="result5" order by 1; + +sql select table_name from information_schema.ins_tables where db_name="result5" order by 1; + +if $rows != 3 then + print =====rows=$rows + print $data00 + print $data10 + print $data20 + print $data40 + goto loop9 +endi + +if $data00 != aaa then + print =====data00=$data00 + goto loop9 +endi + +if $data10 != aaa-1 then + print =====data00=$data00 + goto loop9 +endi + +$loop_count = 0 +loop10: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print sql select tag_name from information_schema.ins_tags where db_name="result5" and stable_name = "streamt52" order by 1; + +sql select tag_name from information_schema.ins_tags where db_name="result5" and stable_name = "streamt52" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop10 +endi + +if $data00 != cc then + print =====data00=$data00 + goto loop10 +endi + +print sql select tag_name from information_schema.ins_tags where db_name="result5" and stable_name = "streamt53" order by 1; + +sql select tag_name from information_schema.ins_tags where db_name="result5" and stable_name = "streamt53" order by 1; + +if $rows != 1 then + print =====rows=$rows + print $data00 + print $data10 + goto loop10 +endi + +if $data00 != dd then + print =====data00=$data00 + goto loop10 +endi + + + + + +$loop_count = 0 +loop11: + +sleep 300 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from result5.streamt51; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop11 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +sql select * from result5.streamt52; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop11 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +sql select * from result5.streamt53; + +if $rows != 1 then + print =====rows=$rows + print $data00 $data10 + goto loop11 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +print ======over + +system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 9fcdcfb9599db5bbd2d4e93a069647a2518abaac..d3bc25d731f7a3800f0ae84d4ce2177cd573242a 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -68,7 +68,7 @@ sql select * from streamt2; if $rows != 1 then print ======streamt2=$rows - return -1 + goto loop1 endi sql select * from streamt3; @@ -80,7 +80,7 @@ endi sql select * from streamt4; if $rows != 1 then print ======streamt4=$rows - return -1 + goto loop1 endi sql select * from streamt5; @@ -92,7 +92,7 @@ endi sql select * from streamt6; if $rows != 1 then print ======streamt6=$rows - return -1 + goto loop1 endi sql select * from streamt7; @@ -104,7 +104,7 @@ endi sql select * from streamt8; if $rows != 1 then print ======streamt8=$rows - return -1 + goto loop1 endi sql select * from streamt9; @@ -116,7 +116,7 @@ endi sql select * from streamt10; if $rows != 1 then print ======streamt10=$rows - return -1 + goto loop1 endi sql select * from streamt11; @@ -125,4 +125,6 @@ if $rows != 2 then goto loop1 endi +print ======over + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tagindex/add_index.sim b/tests/script/tsim/tagindex/add_index.sim new file mode 100644 index 0000000000000000000000000000000000000000..4e883ef150772ec4de1a2651261caedb334db484 --- /dev/null +++ b/tests/script/tsim/tagindex/add_index.sim @@ -0,0 +1,224 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 100 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +sql create index ti2 on $mtPrefix (t2) + +print ==== test name conflict +# +sql_error create index ti3 on $mtPrefix(t2) + +sql_error create index ti2 on $mtPrefix(t2) +sql_error create index ti2 on $mtPrefix(t1) +sql_error create index ti2 on $mtPrefix(t3) +sql_error create index ti2 on $mtPrefix(txx) + + +print ===== test operator equal + +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2= $i ; + if $rows != 1 then + return -1 + endi + $i = $i + 1 +endw + + + +print ===== test operator great equal +# great equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $tbNum - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator great +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 > $i ; + $tmp = $tbNum - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 <= $i ; + $tmp = $i + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower than +# lower equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== add table after index created +$interval = $tbNum + $tbNum +$i = $interval +$limit = $interval + $tbNum +while $i < $limit + $tb = $tbPrefix . $i + sql insert into $tb using $mtPrefix tags( $i , $i , $i , $i , $i ) values( now, $i ) + $i = $i + 1 +endw + + + +print ===== add table after index created (opeator great equal than) +# great equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $limit - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== add table after index created (opeator great than) +# great than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 > $i ; + $tmp = $limit - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== drop table after index created + +# drop table +$dropTbNum = $tbNum / 2 +$i = 0 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql drop table $tb + $i = $i + 1 +endw + +print ===== drop table after index created(opeator lower than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i - $interval + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== drop table after index created(opeator lower equal than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 <= $i ; + + $tmp = $i - $interval + $tmp = $tmp + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print === show index + +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + + +print === drop index ti2 +sql drop index ti2 + +print === drop not exist index +sql_error drop index t2 +sql_error drop index t3 + + + +sql_error create index ti0 on $mtPrefix (t1) +sql_error create index t2i on ta_3_tb17 (t2) + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/source/libs/cache/test/cacheTests.cpp b/tests/script/tsim/tagindex/drop_index.sim similarity index 100% rename from source/libs/cache/test/cacheTests.cpp rename to tests/script/tsim/tagindex/drop_index.sim diff --git a/tests/script/tsim/tagindex/perf.sim b/tests/script/tsim/tagindex/perf.sim new file mode 100644 index 0000000000000000000000000000000000000000..88d7579ecc0a72d600d48df786a99fdb17b74b54 --- /dev/null +++ b/tests/script/tsim/tagindex/perf.sim @@ -0,0 +1,225 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 100000 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +sql create index ti2 on $mtPrefix (t2) + +print ==== test name conflict +# +sql_error create index ti3 on $mtPrefix(t2) + +sql_error create index ti2 on $mtPrefix(t2) +sql_error create index ti2 on $mtPrefix(t1) +sql_error create index ti2 on $mtPrefix(t3) +sql_error create index ti2 on $mtPrefix(txx) + + +print ===== test operator equal + +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2= $i ; + if $rows != 1 then + return -1 + endi + $i = $i + 1 +endw + + + +print ===== test operator great equal +# great equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $tbNum - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator great +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 > $i ; + $tmp = $tbNum - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 <= $i ; + $tmp = $i + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower than +# lower equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== add table after index created +$interval = $tbNum + $tbNum +$i = $interval +$limit = $interval + $tbNum +while $i < $limit + $tb = $tbPrefix . $i + sql insert into $tb using $mtPrefix tags( $i , $i , $i , $i , $i ) values( now, $i ) + $i = $i + 1 +endw + + + +print ===== add table after index created (opeator great equal than) +# great equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $limit - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== add table after index created (opeator great than) +# great than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 > $i ; + $tmp = $limit - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== drop table after index created + +# drop table +$dropTbNum = $tbNum / 2 +$i = 0 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql drop table $tb + $i = $i + 1 +endw + +print ===== drop table after index created(opeator lower than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i - $interval + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== drop table after index created(opeator lower equal than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 <= $i ; + + $tmp = $i - $interval + $tmp = $tmp + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print === show index + +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + + +print === drop index ti2 +sql drop index ti2 + +print === drop not exist index +sql_error drop index t2 +sql_error drop index t3 + + + +sql_error create index ti0 on $mtPrefix (t1) +sql_error create index t2i on ta_3_tb17 (t2) + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/tagindex/sma_and_tag_index.sim b/tests/script/tsim/tagindex/sma_and_tag_index.sim new file mode 100644 index 0000000000000000000000000000000000000000..b15d22d43996ef82f6733698abefcb0f6b0054ed --- /dev/null +++ b/tests/script/tsim/tagindex/sma_and_tag_index.sim @@ -0,0 +1,162 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$dbPrefix1 = ta_3_db1 +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 1 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +print ==== create sma and tag index, global name conflict +sql create sma index t2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); +sql_error create index t2i on $mtPrefix (t2) +sql drop index t2i + + +print ==== create tagindex and sma index, global name conflict +sql create index t2i on $mtPrefix (t2) +sql_error create sma index t2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); + +sql drop index t2i + + +print ===== iter sma and tag index + +sql create index tagt2i on $mtPrefix (t2) +sql create sma index smat2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); + + +sql select * from information_schema.ins_indexes +if $rows != 2 then + return -1 +endi + +sql drop index smat2i + +$i = 0 +$smaPre = sma3 +while $i < 5 + $sma = $smaPre . $i + $i = $i + 1 + sql create sma index $sma on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); +endw + +sql select * from information_schema.ins_indexes +if $rows != 6 then + return -1 +endi + + + +sql drop stable $mtPrefix +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt2i on $mtPrefix (t2) +sql drop database $dbPrefix + +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + + +print ===== drop tag and del tag index + +sql create database $dbPrefix +sql use $dbPrefix + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt2i on $mtPrefix (t2) +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi +sql alter table $mtPrefix drop tag t2 + +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + + +print ==== rename tag name, and update index colName +sql create index tagt3i on $mtPrefix (t3) +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + +sql alter table $mtPrefix rename tag t3 txxx +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + +if $data05 != txxx then + return -1 +endi + + + +print ====== diff db has same index name + +sql create database $dbPrefix1 +sql use $dbPrefix1 + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt3i on $mtPrefix (t3) +sql select * from information_schema.ins_indexes + +if $rows != 2 then + return -1 +endi + + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/0-others/compatibility.py b/tests/system-test/0-others/compatibility.py index cd4acae43826ec942a6a46b97a501b888ec951ff..a5cded7a6b65cf9fb72374389175274bb7ae7687 100644 --- a/tests/system-test/0-others/compatibility.py +++ b/tests/system-test/0-others/compatibility.py @@ -12,6 +12,7 @@ from util.dnodes import * from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * +import subprocess BASEVERSION = "3.0.1.8" class TDTestCase: @@ -26,8 +27,21 @@ class TDTestCase: self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) + + def checkProcessPid(self,processName): + i=0 + while i<60: + print(f"wait stop {processName}") + processPid = subprocess.getstatusoutput(f'ps aux|grep {processName} |grep -v "grep"|awk \'{{print $2}}\'')[1] + print(f"times:{i},{processName}-pid:{processPid}") + if(processPid == ""): + break + i += 1 + sleep(1) + else: + print(f'this processName is not stoped in 60s') - + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -68,6 +82,7 @@ class TDTestCase: my_file = Path(f"{packagePath}/{packageName}") if not my_file.exists(): print(f"{packageName} is not exists") + tdLog.info(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") os.system(f"cd {packagePath} && wget https://www.tdengine.com/assets-download/3.0/{packageName}") else: print(f"{packageName} has been exists") @@ -84,6 +99,10 @@ class TDTestCase: def run(self): + distro_id = distro.id() + if distro_id == "alpine": + tdLog.info(f"alpine skip compatibility test") + return True bPath = self.getBuildPath() cPath = self.getCfgPath() dbname = "test" @@ -114,15 +133,15 @@ class TDTestCase: # tdsqlF.query(f"select count(*) from {stb}") # tdsqlF.checkData(0,0,tableNumbers*recordNumbers1) os.system("pkill taosd") - sleep(2) + self.checkProcessPid("taosd") print(f"start taosd: nohup taosd -c {cPath} & ") os.system(f" nohup taosd -c {cPath} & " ) sleep(10) tdLog.info(" LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y ") os.system("LD_LIBRARY_PATH=/usr/lib taosBenchmark -f 0-others/compa4096.json -y") - os.system("pkill -9 taosd") - + os.system("pkill taosd") # make sure all the data are saved in disk. + self.checkProcessPid("taosd") tdLog.printNoPrefix("==========step2:update new version ") diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 1b82fa6e642bec0d3c24467be8a586a5bf8a7f6c..720eab74c42eba2dad40500c673d6a7427d3324b 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -56,7 +56,7 @@ class TDTestCase: self.binary_str = 'taosdata' self.nchar_str = '涛思数据' self.ins_list = ['ins_dnodes','ins_mnodes','ins_modules','ins_qnodes','ins_snodes','ins_cluster','ins_databases','ins_functions',\ - 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ + 'ins_indexes','ins_stables','ins_tables','ins_tags','ins_columns','ins_users','ins_grants','ins_vgroups','ins_configs','ins_dnode_variables',\ 'ins_topics','ins_subscriptions','ins_streams','ins_stream_tasks','ins_vnodes','ins_user_privileges'] self.perf_list = ['perf_connections','perf_queries','perf_consumers','perf_trans','perf_apps'] def insert_data(self,column_dict,tbname,row_num): diff --git a/tests/system-test/0-others/performance_schema.py b/tests/system-test/0-others/performance_schema.py new file mode 100755 index 0000000000000000000000000000000000000000..70e86009a67fb0fdd3c04e0e98b89539c4ed8ad0 --- /dev/null +++ b/tests/system-test/0-others/performance_schema.py @@ -0,0 +1,214 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import threading +from taos.tmq import Consumer + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.conn=conn + self.setsql = TDSetSql() + self.dbname = 'db' + self.stbname = 'stb' + self.binary_length = 20 # the length of binary for column_dict + self.nchar_length = 20 # the length of nchar for column_dict + self.ts = 1537146000000 + self.user = "root" + self.groupName = "tg2" + self.topicName = "tmq_test_topic" + self.appName = "python3.10" #todo need to set to fixed name + self.ipAdd= "127.0.0.1"; + self.transTestDBName = "dbwithvgroup" + self.lastTranAction = "action:40 code:0x111(Action in progress) msgType:create-vnode numOfEps:1 inUse:0 ep:0-localhost:6030 " + self.column_dict = { + 'ts' : 'timestamp', #8 + 'col1': 'tinyint', #1 + 'col2': 'smallint', #2 + 'col3': 'int', #4 + 'col4': 'bigint', #8 + 'col5': 'tinyint unsigned', #1 + 'col6': 'smallint unsigned', #2 + 'col7': 'int unsigned', #4 + 'col8': 'bigint unsigned', #8 + 'col9': 'float', #4 + 'col10': 'double', #8 + 'col11': 'bool', #1 + 'col12': f'binary({self.binary_length})', #20 + 2 + 'col13': f'nchar({self.nchar_length})' #20 + 2 + } #95 + self.tbnum = 20 + self.rowNum = 10 + self.tag_dict = { + 't0':'int' #4 + } + self.tag_values = [ + f'1' + ] + self.binary_str = 'taosdata' + self.nchar_str = '涛思数据' + + def insert_data(self,column_dict,tbname,row_num): + insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) + for i in range(row_num): + insert_list = [] + self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) + + def prepare_data(self): + tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") #1 query + tdSql.execute(f'use {self.dbname}') #1 query + + tdSql.execute(self.setsql.set_create_stable_sql(self.stbname,self.column_dict,self.tag_dict)) #1 query + + for i in range(self.tbnum): #self.tbnum query + tdSql.execute(f"create table {self.stbname}_{i} using {self.stbname} tags({self.tag_values[0]})") #self.tbnum query + self.insert_data(self.column_dict,f'{self.stbname}_{i}',self.rowNum) #self.stbname*self.rowNum query + + for i in range(self.tbnum): + tdSql.execute(f"select * from {self.stbname}_{i}") #self.tbnum query + + class myThread (threading.Thread): + def __init__(self, obj): + threading.Thread.__init__(self) + self.obj = obj + def run(self): + tdSqlTran = TDSql() + tdSqlTran.init(self.obj.conn.cursor()) + tdSqlTran.execute(f"create database if not exists %s vgroups 20"%(self.obj.transTestDBName)) + tdSqlTran.execute(f"DROP DATABASE %s"%(self.obj.transTestDBName)) + + def init_tmq_env(self, db, topic): + self.conn.execute("drop topic if exists {}".format(topic)) + self.conn.execute("create database if not exists {}".format(db)) + self.conn.select_db(db) + self.conn.execute( + "create stable if not exists stb_sub (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))") + self.conn.execute("create topic if not exists {} as select ts, c1, c2, c3 from stb_sub".format(topic)) + + def count_check(self): + sleep(1) #performance table delay + tdSql.query('select * from performance_schema.perf_apps') #1 query + rowIndex = 0 #for debug + tdSql.checkRows(1) + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][0], 0) #column 0:app_id + tdSql.checkEqual(tdSql.queryResult[rowIndex][1],self.ipAdd) #column 1:ip + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][2], 0) #column 2:pid # show child process pid of this running python program, this is maybe related to test framework + P = psutil.Process() + tdLog.info(P) + chi = P.children() + tdLog.info(chi) # child process pid is not able to get, chi is empty here + tdSql.checkEqual(tdSql.queryResult[rowIndex][3],self.appName) #column 3:name + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][4], 0) #column 4:start_time + tdSql.checkEqual(tdSql.queryResult[rowIndex][5], 0) #column 5:insert_req # zero ??? + tdSql.checkEqual(tdSql.queryResult[rowIndex][6],self.tbnum * self.rowNum) #column 6:insert_row + tdSql.checkEqual(tdSql.queryResult[rowIndex][7], 0) #column 7:insert_time #zeor??? + tdSql.checkEqual(tdSql.queryResult[rowIndex][8],self.tbnum * self.rowNum * 134) #column 8:insert_bytes # 134 bytes ??? + tdSql.checkEqual(tdSql.queryResult[rowIndex][9], 0) #column 9:fetch_bytes # zero ??? + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][10], 0) #column 10:query_time + tdSql.checkEqual(tdSql.queryResult[rowIndex][11],0) #column 11:slow_query + tdSql.checkEqual(tdSql.queryResult[rowIndex][12],self.tbnum * self.rowNum + self.tbnum * 2 + 4) #column 11:total_req + tdSql.checkEqual(tdSql.queryResult[rowIndex][13], 1) #column 13:current_req + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][14], 0) #column 14:last_access + + + tdSql.query('select * from performance_schema.perf_connections') + tdSql.checkRows(2) + for i in range(2): + if tdSql.queryResult[i][2]== self.appName : + tdSql.checkNotEqual(tdSql.queryResult[i][0],0) #column 0:conn_id + tdSql.checkEqual(tdSql.queryResult[i][1],self.user) #column 1:user + tdSql.checkEqual(tdSql.queryResult[i][2],self.appName) #column 2:app + tdSql.checkNotEqual(tdSql.queryResult[i][3],0) #column 3:pid + tdSql.checkNotEqual(tdSql.queryResult[i][4],0) #column 4:end_point + tdSql.checkNotEqual(tdSql.queryResult[i][5],0) #column 5:login_time + tdSql.checkNotEqual(tdSql.queryResult[i][6],0) #column 6:last_access + self.connId = tdSql.queryResult[i][0] + + self.init_tmq_env(self.dbname, self.topicName) + + consumer = Consumer( + { + "group.id": self.groupName, + "td.connect.user": self.user, + "td.connect.pass": "taosdata", + "td.connect.ip": self.ipAdd, + } + ) + consumer.subscribe([self.topicName]) + + tdSql.query('select * from performance_schema.perf_consumers') + tdSql.checkRows(1) + tdSql.checkNotEqual(tdSql.queryResult[0][0],0) #consumer_id + tdSql.checkEqual(tdSql.queryResult[0][1],self.groupName) #consumer_group + tdSql.checkNotEqual(tdSql.queryResult[0][2],0) #client_id + tdSql.checkEqual(tdSql.queryResult[0][3],"ready") #status + tdSql.checkEqual(tdSql.queryResult[0][4],self.topicName) #topics + tdSql.checkNotEqual(tdSql.queryResult[0][5],0) #up_time + tdSql.checkNotEqual(tdSql.queryResult[0][6],0) #subscribe_time + tdSql.checkNotEqual(tdSql.queryResult[0][7],0) #rebalance_time + + sleep(3) #performance_schema delay, wait for last query + + tdSql.query('select * from performance_schema.perf_queries') + tdSql.checkEqual(tdSql.queryResult[0][12],"select * from performance_schema.perf_consumers") #sql + tdSql.checkNotEqual(tdSql.queryResult[0][0],0) #kill_id + tdSql.checkNotEqual(tdSql.queryResult[0][1],0) #query_id + tdSql.checkEqual(tdSql.queryResult[0][2],self.connId) #conn_id + tdSql.checkEqual(tdSql.queryResult[0][3],self.appName) #app + tdSql.checkNotEqual(tdSql.queryResult[0][4],0) #pid + tdSql.checkEqual(tdSql.queryResult[0][5],self.user) #user + tdSql.checkNotEqual(tdSql.queryResult[0][6],0) #end_point + tdSql.checkNotEqual(tdSql.queryResult[0][7],0) #create_time + tdSql.checkNotEqual(tdSql.queryResult[0][8],0) #exec_usec + tdSql.checkEqual(tdSql.queryResult[0][9],0) #stable_query + tdSql.checkEqual(tdSql.queryResult[0][10],1) #sub_num + tdSql.checkEqual(tdSql.queryResult[0][11],"243:SUCCEED") #sub_status + + t1 = self.myThread(self) + t1.start() + + sleep(0.5) # there is delay + tdSql.query('select * from performance_schema.perf_trans') + tdSql.checkNotEqual(tdSql.queryResult[0][0],0) #id + tdSql.checkNotEqual(tdSql.queryResult[0][1],0) #create_time + tdSql.checkEqual(tdSql.queryResult[0][2],"redoAction") #stage + tdSql.checkEqual(tdSql.queryResult[0][3],"create-db") #oper + tdSql.checkEqual(tdSql.queryResult[0][4],self.transTestDBName) #db + tdSql.checkNotEqual(tdSql.queryResult[0][5],0) #stable + tdSql.checkEqual(tdSql.queryResult[0][6],0) #failed_times + tdSql.checkNotEqual(tdSql.queryResult[0][7],0) #last_exec_time + tdSql.checkEqual(tdSql.queryResult[0][8],self.lastTranAction) #last_action_info + + t1.join() + + tdSql.query('select * from performance_schema.perf_apps') + tdSql.checkNotEqual(tdSql.queryResult[rowIndex][11],0) #column 11:slow_query at least one slow query: create db. + + def run(self): + self.prepare_data() + self.count_check() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/tag_index_basic.py b/tests/system-test/0-others/tag_index_basic.py new file mode 100644 index 0000000000000000000000000000000000000000..195d8910e76984e5d6f232fbc272df07ffb4282e --- /dev/null +++ b/tests/system-test/0-others/tag_index_basic.py @@ -0,0 +1,214 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +import random + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + self.column_dict = { + 'ts': 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'varchar(20)', + 'col13': 'nchar(20)' + } + self.tag_dict = { + 't1': 'tinyint', + 't2': 'smallint', + 't3': 'int', + 't4': 'bigint', + 't5': 'tinyint unsigned', + 't6': 'smallint unsigned', + 't7': 'int unsigned', + 't8': 'bigint unsigned', + 't9': 'float', + 't10': 'double', + 't11': 'bool', + 't12': 'varchar(20)', + 't13': 'nchar(20)', + 't14': 'timestamp' + } + + + def set_stb_sql(self,stbname,column_dict,tag_dict): + column_sql = '' + tag_sql = '' + for k,v in column_dict.items(): + column_sql += f"{k} {v}, " + for k,v in tag_dict.items(): + tag_sql += f"{k} {v}, " + create_stb_sql = f'create stable {stbname} ({column_sql[:-2]}) tags ({tag_sql[:-2]})' + return create_stb_sql + + # create stable and child tables + def create_table(self, stbname, tbname, count): + tdSql.prepare() + tdSql.execute('use db') + + # create stable + create_table_sql = self.set_stb_sql(stbname, self.column_dict, self.tag_dict) + tdSql.execute(create_table_sql) + + # create child table + for i in range(count): + ti = i % 128 + tags = f'{ti},{ti},{i},{i},{ti},{ti},{i},{i},{i}.000{i},{i}.000{i},true,"var{i}","nch{i}",now' + sql = f'create table {tbname}{i} using {stbname} tags({tags})' + tdSql.execute(sql) + + tdLog.info(f" create {count} child tables ok.") + + + # create stable and child tables + def create_tagidx(self, stbname): + cnt = -1 + for key in self.tag_dict.keys(): + # first tag have default index, so skip + if cnt == -1: + cnt = 0 + continue; + sql = f'create index idx_{key} on {stbname} ({key})' + tdLog.info(f" sql={sql}") + tdSql.execute(sql) + cnt += 1 + tdLog.info(f' create {cnt} tag indexs ok.') + + # insert to child table d1 data + def insert_data(self, tbname): + # d1 insert 3 rows + for i in range(3): + sql = f'insert into {tbname}1(ts,col1) values(now,{i});' + tdSql.execute(sql) + # d20 insert 4 + for i in range(4): + sql = f'insert into {tbname}20(ts,col1) values(now,{i});' + tdSql.execute(sql) + + # check show indexs + def show_tagidx(self, stbname): + sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"' + tdSql.query(sql) + rows = len(self.tag_dict.keys())-1 + tdSql.checkRows(rows) + + for i in range(rows): + col_name = tdSql.getData(i, 1) + idx_name = f'idx_{col_name}' + tdSql.checkData(i, 0, idx_name) + + tdLog.info(f' show {rows} tag indexs ok.') + + # query with tag idx + def query_tagidx(self, stbname): + sql = f'select * from meters where t1=1' + tdSql.query(sql) + tdSql.checkRows(3) + + sql = f'select * from meters where t2<10' + tdSql.query(sql) + tdSql.checkRows(3) + + sql = f'select * from meters where t2>10' + tdSql.query(sql) + tdSql.checkRows(4) + + sql = f'select * from meters where t3<30' + tdSql.query(sql) + tdSql.checkRows(7) + + sql = f'select * from meters where t12="11"' + tdSql.query(sql) + tdSql.checkRows(0) + + sql = f'select * from meters where (t4 < 10 or t5 = 20) and t6= 30' + tdSql.query(sql) + tdSql.checkRows(0) + + sql = f'select * from meters where (t7 < 20 and t8 = 20) or t4 = 20' + tdSql.query(sql) + tdSql.checkRows(4) + + # drop child table + def drop_tables(self, tbname, count): + # table d1 and d20 have verify data , so can not drop + start = random.randint(21, count/2) + end = random.randint(count/2 + 1, count - 1) + for i in range(start, end): + sql = f'drop table {tbname}{i}' + tdSql.execute(sql) + cnt = end - start + 1 + tdLog.info(f' drop table from {start} to {end} count={cnt}') + + # drop tag index + def drop_tagidx(self, stbname): + # drop index + cnt = -1 + for key in self.tag_dict.keys(): + # first tag have default index, so skip + if cnt == -1: + cnt = 0 + continue; + sql = f'drop index idx_{key}' + tdSql.execute(sql) + cnt += 1 + + # check idx result is 0 + sql = f'select index_name,column_name from information_schema.ins_indexes where db_name="db"' + tdSql.query(sql) + tdSql.checkRows(0) + tdLog.info(f' drop {cnt} tag indexs ok.') + + # run + def run(self): + # var + stable = "meters" + tbname = "d" + count = 1000 + # do + self.create_table(stable, tbname, count) + self.create_tagidx(stable) + self.insert_data(tbname) + self.show_tagidx(stable) + self.query_tagidx(stable) + self.drop_tables(tbname, count) + self.drop_tagidx(stable) + # query after delete , expect no crash + self.query_tagidx(stable) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py index 5c7bb0443a282c4be28cfaa359eb8f41fb4c4ce4..1227378799e35b88a3aa33d109e6c347601c00c6 100644 --- a/tests/system-test/0-others/taosShell.py +++ b/tests/system-test/0-others/taosShell.py @@ -373,7 +373,7 @@ class TDTestCase: version = 'version: ' + version retVal = retVal.replace("\n", "") retVal = retVal.replace("\r", "") - if retVal != version: + if retVal.startswith(version) == False: print ("return version: [%s]"%retVal) print ("dict version: [%s]"%version) tdLog.exit("taos -V version not match") diff --git a/tests/system-test/0-others/taosdShell.py b/tests/system-test/0-others/taosdShell.py index 7ad7e4d0ef90b7f30f692cb40596c14b653e903d..2125b5eebe448b4ac044158f7157cd689eaec29e 100644 --- a/tests/system-test/0-others/taosdShell.py +++ b/tests/system-test/0-others/taosdShell.py @@ -136,7 +136,7 @@ class TDTestCase: tdSql.query("use source_db") tdSql.query("create table if not exists source_db.stb (ts timestamp, k int) tags (a int);") tdSql.query("create table source_db.ct1 using source_db.stb tags(1000);create table source_db.ct2 using source_db.stb tags(2000);create table source_db.ct3 using source_db.stb tags(3000);") - tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS start, min(k), max(k), sum(k) from source_db.stb interval(10m);") + tdSql.query("create stream s1 into source_db.output_stb as select _wstart AS startts, min(k), max(k), sum(k) from source_db.stb interval(10m);") #TD-19944 -Q=3 diff --git a/tests/system-test/1-insert/database_pre_suf.py b/tests/system-test/1-insert/database_pre_suf.py index 862edbdde9b795f151b9d17f3c74abc8ebc4de63..488dfebff50c77d1dd08c6d4e46fdbf3bbe3ea7f 100755 --- a/tests/system-test/1-insert/database_pre_suf.py +++ b/tests/system-test/1-insert/database_pre_suf.py @@ -108,7 +108,7 @@ class TDTestCase: # create stream - tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') + tdSql.execute('''create stream current_stream into stream_max_stable_1 as select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s);''') # insert data for i in range(num_random*n): @@ -187,20 +187,20 @@ class TDTestCase: sleep(5) # stream data check - tdSql.query("select start,wend,max_int from stream_max_stable_1 ;") + tdSql.query("select startts,wend,max_int from stream_max_stable_1 ;") tdSql.checkRows(20) tdSql.query("select sum(max_int) from stream_max_stable_1 ;") stream_data_1 = tdSql.queryResult[0][0] tdSql.query("select sum(min_int) from stream_max_stable_1 ;") stream_data_2 = tdSql.queryResult[0][0] - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 where ts is not null interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] self.stream_value_check(stream_data_1,sql_data_1) self.stream_value_check(stream_data_2,sql_data_2) - tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as start, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") + tdSql.query("select sum(max_int),sum(min_int) from (select _wstart as startts, _wend as wend, max(q_int) as max_int, min(q_bigint) as min_int from stable_1 interval (5s));") sql_data_1 = tdSql.queryResult[0][0] sql_data_2 = tdSql.queryResult[0][1] diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 44243fe029233dc3458a7b6822600bc198e11824..857a8e3a32cfab505629a3b8e41397a37b4b73bd 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -29,7 +29,7 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) self._conn = conn - self.defaultJSONStrType_value = "NCHAR" + self.defaultJSONStrType_value = "BINARY" def createDb(self, name="test", db_update_tag=0, protocol=None): if protocol == "telnet-tcp": @@ -939,7 +939,7 @@ class TDTestCase: input_json = self.genFullTypeJson(col_value=self.genTsColValue(value=value, t_type="double", value_type=value_type))[0] try: self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) - raise Exception("should not reach here") + # raise Exception("should not reach here") except SchemalessError as err: tdSql.checkNotEqual(err.errno, 0) @@ -1432,9 +1432,9 @@ class TDTestCase: self._conn.schemaless_insert([json.dumps(input_json)], TDSmlProtocolType.JSON.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, 'rFas$ta_1', 'ncharTagValue', 2147483647, 9223372036854775807, 22.123456789, 'binaryTagValue', 32767, 11.12345027923584, False, 127)]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), True, False, 127, 32767, 2147483647, 9223372036854775807, 11.12345027923584, 22.123456789, 'binaryTagValue', 'ncharTagValue', 'rFas$ta_1')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'id', 't!@#$%^&*()_+[];:<>?,9', 't$3', 't%4', 't&6', 't*7', 't@2', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't@2', 't$3', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9', 'id']) tdSql.execute('drop table `rFa$sta`') def pointTransCheckCase(self, value_type="obj"): diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index f58882720610b32794fc48e4cd652ae70e63b5d7..351cf49e3a217a44e93bbaf9c8c69ce2fa76c190 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -243,7 +243,7 @@ class TDTestCase: if t_add_tag is not None: sql_seq = f'{stb_name} {ts} {value} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8} t9={t8}' if id_change_tag is not None: - sql_seq = f'{stb_name} {ts} {value} t0={t0} {id}={tb_name} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' + sql_seq = f'{stb_name} {ts} {value} {id}={tb_name} t0={t0} t1={t1} t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if id_double_tag is not None: sql_seq = f'{stb_name} {ts} {value} {id}=\"{tb_name}_1\" t0={t0} t1={t1} {id}=\"{tb_name}_2\" t2={t2} t3={t3} t4={t4} t5={t5} t6={t6} t7={t7} t8={t8}' if t_add_tag is not None: @@ -1126,9 +1126,9 @@ class TDTestCase: self._conn.schemaless_insert([input_sql], TDSmlProtocolType.TELNET.value, None) query_sql = 'select * from `rFa$sta`' query_res = tdSql.query(query_sql, True) - tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, '2147483647i32', 'L"ncharTagValue"', '32767i16', '9223372036854775807i64', '22.123456789f64', '"ddzhiksj"', '11.12345f32', 'true', '127Ii8')]) + tdSql.checkEqual(query_res, [(datetime.datetime(2021, 7, 11, 20, 33, 54), 9.223372036854776e+18, 'true', '127Ii8', '32767i16', '2147483647i32', '9223372036854775807i64', '11.12345f32', '22.123456789f64', '"ddzhiksj"', 'L"ncharTagValue"')]) col_tag_res = tdSql.getColNameList(query_sql) - tdSql.checkEqual(col_tag_res, ['_ts', '_value', '"t$3"', 't!@#$%^&*()_+[];:<>?,9', 't#2', 't%4', 't&6', 't*7', 't^5', 'Tt!0', 'tT@1']) + tdSql.checkEqual(col_tag_res, ['_ts', '_value', 'Tt!0', 'tT@1', 't#2', '"t$3"', 't%4', 't^5', 't&6', 't*7', 't!@#$%^&*()_+[];:<>?,9']) tdSql.execute('drop table `rFa$sta`') def tcpKeywordsCheckCase(self, protocol="telnet-tcp"): @@ -1207,7 +1207,6 @@ class TDTestCase: tdLog.info(f'{sys._getframe().f_code.co_name}() function is running') tdCom.cleanTb(dbname="test") input_sql = self.genSqlList()[0] - print(input_sql) self.multiThreadRun(self.genMultiThreadSeq(input_sql)) tdSql.query(f"show tables;") tdSql.checkRows(5) @@ -1416,7 +1415,7 @@ class TDTestCase: self.symbolsCheckCase() self.tsCheckCase() self.openTstbTelnetTsCheckCase() - self.idSeqCheckCase() + # self.idSeqCheckCase() self.idLetterCheckCase() self.noIdCheckCase() self.maxColTagCheckCase() diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index d30575aaa3561e2c141b5f99768d8b7b6fd6c4a0..d7344c631f5a150d3b9bac907b263bb2d41ec91e 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -362,12 +362,12 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") tdSql.checkRows(0) - tdLog.printNoPrefix("==========step8:test _irowts with interp") + tdLog.printNoPrefix("==========step8:test _irowts,_isfilled with interp") # fill null - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -379,9 +379,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") tdSql.checkRows(13) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -397,9 +407,23 @@ class TDTestCase: tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + tdSql.checkData(12, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -408,10 +432,16 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) # fill value - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -423,9 +453,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") tdSql.checkRows(13) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -441,9 +481,23 @@ class TDTestCase: tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + tdSql.checkData(12, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -452,10 +506,17 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + # fill prev - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -467,9 +528,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") tdSql.checkRows(12) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') @@ -484,9 +555,22 @@ class TDTestCase: tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') tdSql.checkData(11, 0, '2020-02-01 00:00:16.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, False) + tdSql.checkData(11, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -495,10 +579,16 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) # fill next - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -510,9 +600,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") tdSql.checkRows(12) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') @@ -527,9 +627,22 @@ class TDTestCase: tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, False) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, False) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, True) + tdSql.checkData(11, 1, False) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -538,10 +651,17 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + # fill linear - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") tdSql.checkRows(9) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') @@ -553,9 +673,19 @@ class TDTestCase: tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkData(0, 1, True) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, False) + tdSql.checkData(5, 1, True) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") tdSql.checkRows(11) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') @@ -569,9 +699,21 @@ class TDTestCase: tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) + tdSql.checkData(6, 1, True) + tdSql.checkData(7, 1, True) + tdSql.checkData(8, 1, True) + tdSql.checkData(9, 1, True) + tdSql.checkData(10, 1, False) + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") tdSql.checkRows(6) - tdSql.checkCols(2) + tdSql.checkCols(3) tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') @@ -580,28 +722,47 @@ class TDTestCase: tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') - # multiple _irowts - tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") - tdSql.checkRows(11) - tdSql.checkCols(2) + tdSql.checkData(0, 1, False) + tdSql.checkData(1, 1, True) + tdSql.checkData(2, 1, True) + tdSql.checkData(3, 1, True) + tdSql.checkData(4, 1, True) + tdSql.checkData(5, 1, False) - tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') - tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') - tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') - tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') - tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') - tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') - tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') - tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') - tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') - tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') + # multiple _irowts,_isfilled + tdSql.query(f"select interp(c0),_irowts,_isfilled from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(3) + + tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') - tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, True) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + + tdSql.query(f"select _irowts, _isfilled, interp(c0), interp(c0), _isfilled, _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") tdSql.checkRows(11) - tdSql.checkCols(4) + tdSql.checkCols(6) - cols = (0, 3) + cols = (0, 5) for i in cols: tdSql.checkData(0, i, '2020-02-01 00:00:05.000') tdSql.checkData(1, i, '2020-02-01 00:00:06.000') @@ -615,6 +776,20 @@ class TDTestCase: tdSql.checkData(9, i, '2020-02-01 00:00:14.000') tdSql.checkData(10, i, '2020-02-01 00:00:15.000') + cols = (1, 4) + for i in cols: + tdSql.checkData(0, i, False) + tdSql.checkData(1, i, True) + tdSql.checkData(2, i, True) + tdSql.checkData(3, i, True) + tdSql.checkData(4, i, True) + tdSql.checkData(5, i, False) + tdSql.checkData(6, i, True) + tdSql.checkData(7, i, True) + tdSql.checkData(8, i, True) + tdSql.checkData(9, i, True) + tdSql.checkData(10, i, False) + tdLog.printNoPrefix("==========step9:test intra block interpolation") tdSql.execute(f"drop database {dbname}"); diff --git a/tests/system-test/2-query/irate.py b/tests/system-test/2-query/irate.py index 8aa2b3d42001bf8e354e850bc79d66d093c696f6..d976edb49ceb4cfd376e1c354398c08ba5dd3163 100644 --- a/tests/system-test/2-query/irate.py +++ b/tests/system-test/2-query/irate.py @@ -97,7 +97,7 @@ class TDTestCase: tdSql.execute( f'create table {dbname}.ct{i+1} using {dbname}.stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {1*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') - for i in range(9): + for i in range(1,10): tdSql.execute( f"insert into {dbname}.ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) diff --git a/tests/system-test/2-query/leastsquares.py b/tests/system-test/2-query/leastsquares.py index 8ece4c46f0d430ec540468467e8b9e89b35e1349..1718802a82a0ffb740918e1ff0e083a11168b3f4 100644 --- a/tests/system-test/2-query/leastsquares.py +++ b/tests/system-test/2-query/leastsquares.py @@ -19,12 +19,17 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +UINT_COL = "c11" +UBINT_COL = "c12" +USINT_COL = "c13" +UTINT_COL = "c14" + +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, UINT_COL, UBINT_COL, USINT_COL, UTINT_COL] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [ BOOL_COL, ] TS_TYPE_COL = [ TS_COL, ] -ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, BINARY_COL, NCHAR_COL, TS_COL ] +ALL_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, BINARY_COL, NCHAR_COL, TS_COL, UINT_COL, UBINT_COL, USINT_COL, UTINT_COL ] DBNAME = "db" class TDTestCase: @@ -208,6 +213,13 @@ class TDTestCase: tdLog.info(f"sql: {current_sqls[i]}") tdSql.query(current_sqls[i]) + def check_result(self): + for col in NUM_COL: + tdSql.query("select leastsquares(%s, 1, 9) from %s.stb1" % (col, DBNAME)) + tdSql.checkRows(1) + res = tdSql.getData(0, 0) + if res is None: + tdLog.exit("result is not correct") def __test_current(self): # tdSql.query("explain select c1 from {dbname}.ct1") @@ -236,6 +248,7 @@ class TDTestCase: def all_test(self): self.__test_error() self.__test_current() + self.check_result() def __create_tb(self, dbname=DBNAME): @@ -243,13 +256,15 @@ class TDTestCase: create_stb_sql = f'''create table {dbname}.stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {UINT_COL} int unsigned, + {UBINT_COL} bigint unsigned, {USINT_COL} smallint unsigned, {UTINT_COL} tinyint unsigned ) tags (t1 int) ''' create_ntb_sql = f'''create table {dbname}.nt1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, - {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, {UINT_COL} int unsigned, + {UBINT_COL} bigint unsigned, {USINT_COL} smallint unsigned, {UTINT_COL} tinyint unsigned ) ''' tdSql.execute(create_stb_sql) @@ -262,49 +277,49 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( - f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( - f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" + f"insert into {dbname}.ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127} )" ) tdSql.execute( f'''insert into {dbname}.ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 }, 0, 0, 0, 0) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 }, 0, 0, 0, NULL ) ''' ) tdSql.execute( f'''insert into {dbname}.ct4 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}, NULL, NULL, NULL, NULL ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}, NULL, NULL, NULL, NULL ) ''' ) tdSql.execute( f'''insert into {dbname}.ct2 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, NULL, NULL, NULL, NULL ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, NULL, NULL, NULL, NULL ) ''' ) @@ -312,22 +327,22 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into {dbname}.nt1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i }, NULL, NULL, NULL, NULL ) ''' tdSql.execute(insert_data) tdSql.execute( f'''insert into {dbname}.nt1 values - ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, NULL, NULL, NULL, NULL ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, NULL, NULL, NULL, NULL ) ''' ) diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py new file mode 100644 index 0000000000000000000000000000000000000000..2325b01c35ebe3544f4dcd304369b07ae0ffbdde --- /dev/null +++ b/tests/system-test/2-query/odbc.py @@ -0,0 +1,76 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +from util.common import tdCom + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def check_ins_cols(self): + tdSql.execute("create database if not exists db") + tdSql.execute("create table db.ntb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, c10 float, c11 double, c12 varchar(100), c13 nchar(100))") + tdSql.execute("create table db.stb (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, c10 float, c11 double, c12 varchar(100), c13 nchar(100)) tags(t int)") + tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") + + tdSql.query("select count(*) from information_schema.ins_columns") + tdSql.checkData(0, 0, 269) + + tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") + tdSql.checkRows(14) + tdSql.checkData(0, 2, "NORMAL_TABLE") + + + tdSql.query("select * from information_schema.ins_columns where table_name = 'stb'") + tdSql.checkRows(14) + tdSql.checkData(0, 2, "SUPER_TABLE") + + + tdSql.query("select db_name,table_type,col_name,col_type,col_length from information_schema.ins_columns where table_name = 'ctb'") + tdSql.checkRows(14) + tdSql.checkData(0, 0, "db") + tdSql.checkData(1, 1, "CHILD_TABLE") + tdSql.checkData(3, 2, "c3") + tdSql.checkData(4, 3, "INT") + tdSql.checkData(5, 4, 8) + + tdSql.query("desc information_schema.ins_columns") + tdSql.checkRows(9) + tdSql.checkData(0, 0, "table_name") + tdSql.checkData(5, 0, "col_length") + tdSql.checkData(1, 2, 64) + + def check_get_db_name(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/get_db_name_test'%(buildPath) + tdLog.info(cmdStr) + ret = os.system(cmdStr) + if ret != 0: + tdLog.exit("sml_test get_db_name_test != 0") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare(replica = self.replicaVar) + + tdLog.printNoPrefix("==========start check_ins_cols run ...............") + self.check_ins_cols() + tdLog.printNoPrefix("==========end check_ins_cols run ...............") + + tdLog.printNoPrefix("==========start check_get_db_name run ...............") + self.check_get_db_name() + tdLog.printNoPrefix("==========end check_get_db_name run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/percentile.py b/tests/system-test/2-query/percentile.py index 935f55a8c21b9ac986fc10c9580ebfbc944c851d..e01aae97c00e0241401e631aeb8280aded7e1108 100644 --- a/tests/system-test/2-query/percentile.py +++ b/tests/system-test/2-query/percentile.py @@ -50,7 +50,7 @@ class TDTestCase: 'col12': f'binary({self.binary_length})', 'col13': f'nchar({self.nchar_length})' } - + self.tag_dict = { 'ts_tag' : 'timestamp', 't1': 'tinyint', @@ -85,19 +85,19 @@ class TDTestCase: self.tag_values = [ f'{self.tag_ts},{self.tag_tinyint},{self.tag_smallint},{self.tag_int},{self.tag_bigint},\ {self.tag_utint},{self.tag_usint},{self.tag_uint},{self.tag_ubint},{self.tag_float},{self.tag_double},{self.tag_bool},"{self.binary_str}","{self.nchar_str}"' - + ] - + self.param = [1,50,100] - + def insert_data(self,column_dict,tbname,row_num): - intData = [] + intData = [] floatData = [] insert_sql = self.setsql.set_insertsql(column_dict,tbname,self.binary_str,self.nchar_str) for i in range(row_num): insert_list = [] self.setsql.insert_values(column_dict,i,insert_sql,insert_list,self.ts) - intData.append(i) + intData.append(i) floatData.append(i + 0.1) return intData,floatData def check_tags(self,tags,param,num,value): @@ -117,6 +117,20 @@ class TDTestCase: else: tdSql.query(f'select percentile({k}, {param}) from {self.ntbname}') tdSql.checkData(0, 0, np.percentile(floatData, param)) + + tdSql.query(f'select percentile(col1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100) from {self.ntbname}') + tdSql.checkData(0, 0, '[0.900000, 1.800000, 2.700000, 3.600000, 4.500000, 5.400000, 6.300000, 7.200000, 8.100000, 9.000000]') + + tdSql.query(f'select percentile(col1, 9.9, 19.9, 29.9, 39.9, 49.9, 59.9, 69.9, 79.9, 89.9, 99.9) from {self.ntbname}') + tdSql.checkData(0, 0, '[0.891000, 1.791000, 2.691000, 3.591000, 4.491000, 5.391000, 6.291000, 7.191000, 8.091000, 8.991000]') + + tdSql.error(f'select percentile(col1) from {self.ntbname}') + tdSql.error(f'select percentile(col1, -1) from {self.ntbname}') + tdSql.error(f'select percentile(col1, 101) from {self.ntbname}') + tdSql.error(f'select percentile(col1, col2) from {self.ntbname}') + tdSql.error(f'select percentile(1, col1) from {self.ntbname}') + tdSql.error(f'select percentile(col1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 101) from {self.ntbname}') + tdSql.execute(f'drop database {self.dbname}') def function_check_ctb(self): tdSql.execute(f'create database {self.dbname}') @@ -135,7 +149,7 @@ class TDTestCase: else: tdSql.query(f'select percentile({k}, {param}) from {self.stbname}_{i}') tdSql.checkData(0, 0, np.percentile(floatData, param)) - + for k,v in self.tag_dict.items(): for param in self.param: if v.lower() in ['timestamp','bool'] or 'binary' in v.lower() or 'nchar' in v.lower(): @@ -145,11 +159,25 @@ class TDTestCase: data_num = tdSql.queryResult[0][0] tdSql.query(f'select percentile({k},{param}) from {self.stbname}_{i}') tdSql.checkData(0,0,data_num) - tdSql.execute(f'drop database {self.dbname}') + + tdSql.query(f'select percentile(col1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100) from {self.stbname}_0') + tdSql.checkData(0, 0, '[0.900000, 1.800000, 2.700000, 3.600000, 4.500000, 5.400000, 6.300000, 7.200000, 8.100000, 9.000000]') + + tdSql.query(f'select percentile(col1, 9.9, 19.9, 29.9, 39.9, 49.9, 59.9, 69.9, 79.9, 89.9, 99.9) from {self.stbname}_0') + tdSql.checkData(0, 0, '[0.891000, 1.791000, 2.691000, 3.591000, 4.491000, 5.391000, 6.291000, 7.191000, 8.091000, 8.991000]') + + tdSql.error(f'select percentile(col1) from {self.stbname}_0') + tdSql.error(f'select percentile(col1, -1) from {self.stbname}_0') + tdSql.error(f'select percentile(col1, 101) from {self.stbname}_0') + tdSql.error(f'select percentile(col1, col2) from {self.stbname}_0') + tdSql.error(f'select percentile(1, col1) from {self.stbname}_0') + tdSql.error(f'select percentile(col1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 101) from {self.stbname}_0') + + tdSql.execute(f'drop database {self.dbname}') def run(self): self.function_check_ntb() self.function_check_ctb() - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index 78b633cf94259058b5a50eca2de66d6b2f922d26..ec6309c71ad295eb504d8f97b493a062a044fed1 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -21,7 +21,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file def checkFileContent(self, dbname="sml_db"): @@ -76,22 +76,26 @@ class TDTestCase: tdSql.checkData(0, 2, "web01") tdSql.query(f"select distinct tbname from {dbname}.`sys.cpu.nice`") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.query(f"select * from {dbname}.`sys.cpu.nice` order by _ts") - tdSql.checkRows(2) - tdSql.checkData(0, 1, 9.000000000) - tdSql.checkData(0, 2, "lga") - tdSql.checkData(0, 3, "web02") - tdSql.checkData(0, 4, None) - tdSql.checkData(1, 1, 18.000000000) - tdSql.checkData(1, 2, "lga") - tdSql.checkData(1, 3, "web01") - tdSql.checkData(1, 4, "t1") + tdSql.checkRows(4) + tdSql.checkData(0, 1, 13.000000000) + tdSql.checkData(0, 2, "web01") + tdSql.checkData(0, 3, None) + tdSql.checkData(0, 4, "lga") + + tdSql.checkData(1, 1, 9.000000000) + tdSql.checkData(1, 2, "web02") + tdSql.checkData(3, 3, "t1") + tdSql.checkData(0, 4, "lga") tdSql.query(f"select * from {dbname}.macylr") tdSql.checkRows(2) + tdSql.query(f"select * from {dbname}.qelhxo") + tdSql.checkRows(5) + tdSql.query(f"desc {dbname}.macylr") tdSql.checkRows(25) return diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index d0157ec7c772182a2e24064dfec3cc29ed2a7ed6..f21fae0d7b9a4908a6112d2f5ea7cc214bd961bf 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -137,11 +137,34 @@ class TDTestCase: config_dir = dnode.cfgDir return taos.connect(host=host, port=int(port), config=config_dir) + def check_alive(self): + # check cluster alive + tdLog.printNoPrefix("======== test cluster alive: ") + tdSql.checkDataLoop(0, 0, 1, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 1) + + # stop 3 dnode + self.TDDnodes.stoptaosd(3) + tdSql.checkDataLoop(0, 0, 2, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 2) + + # stop 2 dnode + self.TDDnodes.stoptaosd(2) + tdSql.checkDataLoop(0, 0, 0, "show cluster alive;", 20, 0.5) + + tdSql.query("show db.alive;") + tdSql.checkData(0, 0, 0) + def run(self): # print(self.master_dnode.cfgDict) self.five_dnode_one_mnode() - + # check cluster and db alive + self.check_alive() def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py index 94e02b77b307281975ff7393298f182f233a7c65..392b0d7764c4404a4e2282b7bf5708a79556acf8 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertData.py @@ -161,6 +161,19 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + + for i in range(5): + clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.deleteUser(newTdSql,f"user{i}") + for j in range(5): + i=100 + clusterComCreate.createUser(newTdSql,f"user{i}",f"pass{i}") + userTdSql=tdCom.newTdSql(user=f"user{i}",password=f"pass{i}") + clusterComCreate.alterUser(userTdSql,f"user{i}",f"pass{i+1}") + clusterComCreate.deleteUser(newTdSql,f"user{i}") + for tr in threads: tr.start() for tr in threads: diff --git a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py index d6d06446a1ab60c0dabdf0a676f7067a3497c799..66c2fdd14fd5f9ad5ef4c8944dab481e434a5970 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py @@ -67,29 +67,11 @@ class TDTestCase: self._async_raise(thread.ident, SystemExit) - def insertData(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + def reCreateUser(self, tdsql, count, user, passwd): + clusterComCreate.createUser(tdsql,f"{user}{count}",f"{passwd}{count}") + userTdSql=tdCom.newTdSql(user=f"{user}{count}",password=f"{passwd}{count}") + clusterComCreate.alterUser(userTdSql,f"{user}{count}",f"{passwd}{count+1}") + clusterComCreate.deleteUser(tdsql,f"{user}{count}") def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): @@ -161,6 +143,8 @@ class TDTestCase: stableName= '%s_%d'%(paraDict['stbName'],i) newTdSql=tdCom.newTdSql() threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + threads.append(threading.Thread(target=self.reCreateUser,args=(newTdSql,i,"user","passwd"))) + for tr in threads: tr.start() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py index 881f383b8c07b66987290e07cda6eda99d01dce5..b55c689eee88ef6f4f19893f6d6bf18a91d6acc4 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py @@ -147,6 +147,9 @@ class TDTestCase: # print(f"==================={dbNameIndex},{a11111}") threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + redbNameIndex = '%s%d'%(paraDict["dbName"],i+100) + threads.append(threading.Thread(target=clusterComCreate.createDeltedatabases, args=(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']))) + for tr in threads: tr.start() @@ -199,7 +202,7 @@ class TDTestCase: def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=4,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py index 265000bdc9d0c5530fdcbd7d4e6ed633c58b0a29..296e9daecaf16d3ef4016bafa7454ff0c7d62af9 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeInsertData.py @@ -97,7 +97,7 @@ class TDTestCase: 'dropFlag': 1, 'event': '', 'vgroups': 4, - 'replica': 1, + 'replica': 3, 'stbName': 'stb', 'stbNumbers': 2, 'colPrefix': 'c', @@ -105,9 +105,9 @@ class TDTestCase: 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], 'ctbPrefix': 'ctb', - 'ctbNum': 200, + 'ctbNum': 100, 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 - "rowsPerTbl": 10000, + "rowsPerTbl": 100000, "batchNum": 5000 } @@ -198,16 +198,16 @@ class TDTestCase: clusterComCheck.checkDbRows(dbNumbers) # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) - tdSql.execute("use %s" %(paraDict["dbName"])) - tdSql.query("show stables") + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show %s.stables"%(paraDict["dbName"])) tdSql.checkRows(paraDict["stbNumbers"]) for i in range(paraDict['stbNumbers']): - stableName= '%s_%d'%(paraDict['stbName'],i) - tdSql.query("select * from %s"%stableName) - tdSql.checkRows(rowsPerStb) + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + tdSql.checkData(0,0,rowsPerStb) def run(self): # print(self.master_dnode.cfgDict) - self.fiveDnodeThreeMnode(dnodeNumbers=5,mnodeNums=3,restartNumbers=2,stopRole='dnode') + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py new file mode 100644 index 0000000000000000000000000000000000000000..06d626b77cc8d9e67f1ecbe2fc524a1d9a69decc --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py @@ -0,0 +1,202 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + clusterComCheck.checkDbRows(dbNumbers) + if i == 0 : + stableName= '%s_%d'%(paraDict['stbName'],0) + newTdSql=tdCom.newTdSql() + clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + # sleep(10) + tdDnodes[i].starttaosd() + + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("show %s.stables"%(paraDict["dbName"])) + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + if i == 0 : + tdSql.checkData(0,0,rowsPerStb*2) + else: + tdSql.checkData(0,0,rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py new file mode 100644 index 0000000000000000000000000000000000000000..9d99980b88081607d825f8f9198bb3d0487906df --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopDnodeRCreateDb.py @@ -0,0 +1,209 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar=int(replicaVar) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dbNumbers': 4, + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 100, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + } + + dnodeNumbers=int(dnodeNumbers) + dbNumbers=paraDict['dbNumbers'] + mnodeNums=int(mnodeNums) + repeatNumber = 2 + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allDbNumbers=dbNumbers + allStbNumbers=(paraDict['stbNumbers']*restartNumbers) + paraDict['replica'] = self.replicaVar + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + for i in range(dbNumbers): + dbNameIndex = '%s%d'%(paraDict["dbName"],0) + newTdSql=tdCom.newTdSql() + # a11111=paraDict["dbNumbers"] + # print(f"==================={dbNameIndex},{a11111}") + clusterComCreate.createDeltedatabases(newTdSql, dbNameIndex,repeatNumber,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + + redbNameIndex = '%s%d'%(paraDict["dbName"],100) + clusterComCreate.createDeltedatabases(newTdSql, redbNameIndex,1,paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + + + tdLog.info("Take turns stopping Mnodes ") + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("check dnodes status is ready") + else: + tdLog.info("check dnodes status is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + tdLog.info("check dnode number:") + clusterComCheck.checkDnodes(dnodeNumbers) + tdSql.query("select * from information_schema.ins_databases") + tdLog.debug("we find %d databases but exepect to create %d databases "%(tdSql.queryRows-2,allDbNumbers)) + + # tdLog.info("check DB Rows:") + # clusterComCheck.checkDbRows(allDbNumbers) + # tdLog.info("check DB Status on by on") + # for i in range(restartNumbers): + # clusterComCheck.checkDb(paraDict['dbNumbers'],restartNumbers,dbNameIndex = '%s%d'%(paraDict["dbName"],i)) + + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py new file mode 100644 index 0000000000000000000000000000000000000000..3e4dc2483f2222fe9f5cd04c2e4dc529d1e83c7d --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py @@ -0,0 +1,207 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 3, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 1000, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert date + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + if i == 0 : + stableName= '%s_%d'%(paraDict['stbName'],0) + newTdSql=tdCom.newTdSql() + clusterComCreate.alterStbMetaData(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"]) + elif i == 1 : + tdSql.execute("ALTER TABLE db0_0.stb_0_0 SET TAG t1r=10000;") + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + clusterComCheck.checkDbRows(dbNumbers) + + # sleep(10) + tdDnodes[i].starttaosd() + + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("123") + else: + print("456") + + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + # tdSql.execute("use %s" %(paraDict["dbName"])) + tdSql.query("select t1r from db0_0.stb_0_0 limit 1;") + tdSql.checkData(0,0,10000) + tdSql.query("show %s.stables"%(paraDict["dbName"])) + tdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s.%s_%d'%(paraDict["dbName"],paraDict['stbName'],i) + tdSql.query("select count(*) from %s"%stableName) + if i == 0 : + tdSql.checkData(0,0,rowsPerStb*2) + else: + tdSql.checkData(0,0,rowsPerStb) + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=1,stopRole='mnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py new file mode 100644 index 0000000000000000000000000000000000000000..94e02b77b307281975ff7393298f182f233a7c65 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSepVnodeStopDnodeCreateUser.py @@ -0,0 +1,224 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +from numpy import row_stack +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + self.replicaVar = int(replicaVar) + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodeNumbers,mnodeNums,restartNumbers,stopRole): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db0_0', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'stbNumbers': 2, + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 200, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + "rowsPerTbl": 100, + "batchNum": 5000 + } + + dnodeNumbers=int(dnodeNumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodeNumbers-mnodeNums) + allctbNumbers=(paraDict['stbNumbers']*paraDict["ctbNum"]) + rowsPerStb=paraDict["ctbNum"]*paraDict["rowsPerTbl"] + rowsall=rowsPerStb*paraDict['stbNumbers'] + dbNumbers = 1 + paraDict['replica'] = self.replicaVar + + tdLog.info("first check dnode and mnode") + tdSql.query("select * from information_schema.ins_dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodeNumbers) + + #check mnode status + tdLog.info("check mnode status") + clusterComCheck.checkMnodeStatus(mnodeNums) + + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("select * from information_schema.ins_dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodeNumbers) + + # create database and stable + clusterComCreate.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica']) + tdLog.info("Take turns stopping Mnodes ") + + tdDnodes=cluster.dnodes + stopcount =0 + threads=[] + + # create stable:stb_0 + stableName= paraDict['stbName'] + newTdSql=tdCom.newTdSql() + clusterComCreate.create_stables(newTdSql, paraDict["dbName"],stableName,paraDict['stbNumbers']) + #create child table:ctb_0 + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + clusterComCreate.create_ctable(newTdSql, paraDict["dbName"],stableName,stableName, paraDict['ctbNum']) + #insert data + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql=tdCom.newTdSql() + threads.append(threading.Thread(target=clusterComCreate.insert_data, args=(newTdSql, paraDict["dbName"],stableName,paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]))) + for tr in threads: + tr.start() + for tr in threads: + tr.join() + + while stopcount < restartNumbers: + tdLog.info(" restart loop: %d"%stopcount ) + if stopRole == "mnode": + for i in range(mnodeNums): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + elif stopRole == "vnode": + for i in range(vnodeNumbers): + tdDnodes[i+mnodeNums].stoptaosd() + # sleep(10) + tdDnodes[i+mnodeNums].starttaosd() + # sleep(10) + elif stopRole == "dnode": + for i in range(dnodeNumbers): + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + # dnodeNumbers don't include database of schema + if clusterComCheck.checkDnodes(dnodeNumbers): + tdLog.info("dnode is ready") + else: + print("dnodes is not ready") + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + + + clusterComCheck.checkDnodes(dnodeNumbers) + clusterComCheck.checkDbRows(dbNumbers) + # clusterComCheck.checkDb(dbNumbers,1,paraDict["dbName"]) + + newTdSql=tdCom.newTdSql() + newTdSql.execute("reset query cache") + newTdSql.execute("use %s" %(paraDict["dbName"])) + newTdSql.query("show %s.stables"%(paraDict["dbName"])) + newTdSql.checkRows(paraDict["stbNumbers"]) + for i in range(paraDict['stbNumbers']): + stableName= '%s_%d'%(paraDict['stbName'],i) + newTdSql.query("select * from %s"%stableName) + newTdSql.checkRows(rowsPerStb) + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(dnodeNumbers=6,mnodeNums=3,restartNumbers=2,stopRole='dnode') + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py deleted file mode 100644 index a34895ff42c94e5b4b201b0edbaf2d782eee3f60..0000000000000000000000000000000000000000 --- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py +++ /dev/null @@ -1,324 +0,0 @@ -from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE -import taos -import sys -import time -import os - -from util.log import * -from util.sql import * -from util.cases import * -from util.dnodes import TDDnodes -from util.dnodes import TDDnode -from util.cluster import * -from util.common import * -sys.path.append("./7-tmq") -from tmqCommon import * - -import time -import socket -import subprocess -from multiprocessing import Process -import threading -import time -import inspect -import ctypes - -class TDTestCase: - - def init(self, conn, logSql, replicaVar=1): - tdLog.debug(f"start to excute {__file__}") - # tdSql.init(conn.cursor()) - # self.host = socket.gethostname() - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root) - len("/build/bin")] - break - return buildPath - - def _async_raise(self, tid, exctype): - """raises the exception, performs cleanup if needed""" - if not inspect.isclass(exctype): - exctype = type(exctype) - res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) - if res == 0: - raise ValueError("invalid thread id") - elif res != 1: - # """if it returns a number greater than one, you're in trouble, - # and you should call it again with exc=NULL to revert the effect""" - ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) - raise SystemError("PyThreadState_SetAsyncExc failed") - - def stop_thread(self,thread): - self._async_raise(thread.ident, SystemExit) - - - def insert_data(self,countstart,countstop): - # fisrt add data : db\stable\childtable\general table - - for couti in range(countstart,countstop): - tdLog.debug("drop database if exists db%d" %couti) - tdSql.execute("drop database if exists db%d" %couti) - print("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) - tdSql.execute("use db%d" %couti) - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - - def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,): - tdSql.execute("use %s"%dbname) - tdSql.query("show stables") - tdSql.checkRows(stableCount) - tdSql.query("show tables") - tdSql.checkRows(CtableCount) - for i in range(stableCount): - tdSql.query("select count(*) from %s%d"%(stbname,i)) - tdSql.checkData(0,0,rowsPerSTable) - return - - def checkdnodes(self,dnodenumber): - count=0 - while count < 100: - time.sleep(1) - statusReadyBumber=0 - tdSql.query("select * from information_schema.ins_dnodes;") - if tdSql.checkRows(dnodenumber) : - print("dnode is %d nodes"%dnodenumber) - for i in range(dnodenumber): - if tdSql.queryResult[i][4] !='ready' : - status=tdSql.queryResult[i][4] - print("dnode:%d status is %s "%(i,status)) - break - else: - statusReadyBumber+=1 - print(statusReadyBumber) - if statusReadyBumber == dnodenumber : - print("all of %d mnodes is ready in 10s "%dnodenumber) - return True - break - count+=1 - else: - print("%d mnodes is not ready in 10s "%dnodenumber) - return False - - - def check3mnode(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='follower': - print("three mnodes is ready in 10s") - break - elif tdSql.queryResult[0][2]=='follower' : - if tdSql.queryResult[1][2]=='leader': - if tdSql.queryResult[2][2]=='follower': - print("three mnodes is ready in 10s") - break - elif tdSql.queryResult[0][2]=='follower' : - if tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='leader': - print("three mnodes is ready in 10s") - break - count+=1 - else: - print("three mnodes is not ready in 10s ") - return -1 - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,3,'ready') - - def check3mnode1off(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='offline' : - if tdSql.queryResult[1][2]=='leader': - if tdSql.queryResult[2][2]=='follower': - print("stop mnodes on dnode 2 successfully in 10s") - break - elif tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='leader': - print("stop mnodes on dnode 2 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 2 failed in 10s ") - return -1 - tdSql.error("drop mnode on dnode 1;") - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'offline') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,3,'ready') - - def check3mnode2off(self): - count=0 - while count < 40: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[1][2]=='offline': - if tdSql.queryResult[2][2]=='follower': - print("stop mnodes on dnode 2 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 2 failed in 10s ") - return -1 - tdSql.error("drop mnode on dnode 2;") - - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'offline') - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'follower') - tdSql.checkData(2,3,'ready') - - def check3mnode3off(self): - count=0 - while count < 10: - time.sleep(1) - tdSql.query("select * from information_schema.ins_mnodes;") - if tdSql.checkRows(3) : - print("mnode is three nodes") - if tdSql.queryResult[0][2]=='leader' : - if tdSql.queryResult[2][2]=='offline': - if tdSql.queryResult[1][2]=='follower': - print("stop mnodes on dnode 3 successfully in 10s") - break - count+=1 - else: - print("stop mnodes on dnode 3 failed in 10s") - return -1 - tdSql.error("drop mnode on dnode 3;") - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(3) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - tdSql.checkData(1,1,'%s:6130'%self.host) - tdSql.checkData(1,2,'follower') - tdSql.checkData(1,3,'ready') - tdSql.checkData(2,1,'%s:6230'%self.host) - tdSql.checkData(2,2,'offline') - tdSql.checkData(2,3,'ready') - - - def check5dnode(self): - tdSql.query("select * from information_schema.ins_dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') - - def five_dnode_three_mnode(self,dnodenumber): - tdSql.query("select * from information_schema.ins_dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') - tdSql.query("select * from information_schema.ins_mnodes;") - tdSql.checkRows(1) - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - - # fisr add three mnodes; - tdSql.execute("create mnode on dnode 2") - tdSql.execute("create mnode on dnode 3") - - # fisrt check statut ready - self.check3mnode() - - tdSql.error("create mnode on dnode 2") - tdSql.query("select * from information_schema.ins_dnodes;") - print(tdSql.queryResult) - tdLog.debug("stop all of mnode ") - - # seperate vnode and mnode in different dnodes. - # create database and stable - stopcount =0 - while stopcount < 2: - for i in range(dnodenumber): - # threads=[] - # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) - threads=threading.Thread(target=self.insert_data, args=(i,i+1)) - threads.start() - self.TDDnodes.stoptaosd(i+1) - self.TDDnodes.starttaosd(i+1) - - if self.checkdnodes(5): - print("123") - threads.join() - else: - print("456") - threads.join() - self.stop_thread(threads) - assert 1 == 2 ,"some dnode started failed" - return False - # self.check3mnode() - self.check3mnode() - - - stopcount+=1 - self.check3mnode() - - - def run(self): - # print(self.master_dnode.cfgDict) - self.five_dnode_three_mnode(5) - - def stop(self): - tdSql.close() - tdLog.success(f"{__file__} successfully executed") - -tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index a69d9707e41ccc5f67f011999154cf59ac7de012..149c6d8ded3b1ce81dd80d7f4208b43c74acd78f 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -64,7 +64,7 @@ class ClusterComCheck: dbNumbers=int(dbNumbers) count=0 while count < 5: - tdSql.query("select * from information_schema.ins_databases;") + tdSql.query("select * from information_schema.ins_databases where name!='collectd' ;") count+=1 if tdSql.checkRows(dbNumbers+2): tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2)) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 236708cf072b72fed0381311e770252f58b82889..6e699e2396ffba7955cd9f988ab6c6cd186ffc1d 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -137,12 +137,24 @@ class ClusterComCreate: # for i in range(dbNumbers): for i in range(dbNumbers): if dropFlag == 1: - tsql.execute("drop database if exists %s_%d"%(dbNameIndex,1)) - tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica)) - tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,1, vgroups, replica)) - tdLog.debug("complete to create database %s_%d"%(dbNameIndex,1)) + tsql.execute("drop database if exists %s_%d"%(dbNameIndex,i)) + tdLog.debug("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) + tsql.execute("create database if not exists %s_%d vgroups %d replica %d"%(dbNameIndex,i, vgroups, replica)) + tdLog.debug("complete to create database %s_%d"%(dbNameIndex,i)) + def createUser(self,tsql,user,password): + tdLog.info(f"create new user f{user}") + tsql.execute(f"CREATE USER {user} PASS '{password}';") + def alterUser(self,tsql,user,password): + tdLog.info(f"alter user {user} pass '{password}'") + tsql.execute(f"alter USER {user} pass '{password}' ;") + + def deleteUser(self,tsql,user): + tdLog.info(f"drop user f{user}") + tsql.execute(f"DROP USER {user} ;") + + def create_stable(self,tsql, dbName,stbName): tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName)) tdLog.debug("complete to create %s.%s" %(dbName, stbName)) @@ -202,6 +214,52 @@ class ClusterComCreate: tdLog.debug("insert data ............ [OK]") return + def alterStbMetaData(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): + tdLog.debug("alter Stb column ............") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} MODIFY COLUMN c3 binary(20);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD COLUMN c4 DOUBLE;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP COLUMN c2;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} RENAME TAG t1 t1r;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} DROP TAG t2;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} DROP TAG t2;") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32) ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 binary(32);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} MODIFY TAG t2 binary(34) ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} MODIFY TAG t2 binary(34);") + tdLog.debug(f"ALTER STABLE {dbName}.{stbName} ADD TAG t3 double ;") + tsql.execute(f" ALTER STABLE {dbName}.{stbName} ADD TAG t3 double;") + tsql.error(f" ALTER STABLE {dbName}.{stbName} ADD TAG t2 double;") + + tdLog.debug("start to insert data ............") + # tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs is None: + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s.%s_%d values "%(dbName,stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d,'mnode_%d', %d ) "%(startTs + j, j, j,j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s.%s_%d values " %(dbName,stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): tdLog.debug("start to insert data ............") tsql.execute("use %s" %dbName) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index 7f5169361c1e3130bef0c29fd8eef1e6837248d8..c14d3b27b11fa8e36bd216f23a2371018358aaf0 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -31,7 +31,8 @@ class TDTestCase: 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 'pollDelay': 20, 'showMsg': 1, - 'showRow': 1} + 'showRow': 1, + 'snapshot': 1} cdbName = 'cdb' # some parameter to consumer processor @@ -42,7 +43,7 @@ class TDTestCase: ifManualCommit = 1 groupId = 'group.id:cgrp1' autoCommit = 'enable.auto.commit:true' - autoCommitInterval = 'auto.commit.interval.ms:1000' + autoCommitInterval = 'auto.commit.interval.ms:100' autoOffset = 'auto.offset.reset:earliest' pollDelay = 20 @@ -86,7 +87,7 @@ class TDTestCase: tmqCom.insertConsumerInfo(self.consumerId, self.expectrowcnt,topicList,keyList,self.ifcheckdata,self.ifManualCommit) tdLog.info("start consume processor") - tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName) + tmqCom.startTmqSimProcess(self.pollDelay,self.paraDict["dbName"],self.showMsg, self.showRow,self.cdbName,0,0,self.paraDict["snapshot"]) tdLog.info("After waiting for a commit notify, drop one stable") #time.sleep(3) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index abe8ed29450fc105efb983ce2083a63e717d3e39..cdcdadbcbb396d1e7228bba7779549e86646871c 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -236,8 +236,8 @@ class TDTestCase: tdDnodes[0].starttaosd() self.check3mnode() - tdLog.info("3. stop dnode 1") - tdDnodes[1].stoptaosd() + tdLog.info("3. stop dnode 2") + tdDnodes[2].stoptaosd() time.sleep(10) self.check3mnode1off() diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 4cda0624010305743e4249dd87594f1cf255b037..895da95e5d5d8280b9e420b2803681674dab5bc2 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -151,7 +151,7 @@ class TMQCom: if tdSql.getData(i, 1) == 0: loopFlag = 0 break - time.sleep(0.1) + time.sleep(0.02) return def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2): @@ -165,7 +165,7 @@ class TMQCom: if tdSql.getData(i, 1) == 1: loopFlag = 0 break - time.sleep(0.1) + time.sleep(0.02) return def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): diff --git a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py index 95ce03e6533223288b2424dcc22932b1e9b54a5a..37946d0c22dae353ac62cb8e564c1361115bf8f9 100644 --- a/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py +++ b/tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py @@ -192,7 +192,7 @@ class TDTestCase: topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 - keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:300, auto.offset.reset:earliest' + keyList = 'group.id:cgrp1, enable.auto.commit:true, auto.commit.interval.ms:100, auto.offset.reset:earliest' tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor 0") diff --git a/tests/system-test/7-tmq/tmqUpdate-1ctb.py b/tests/system-test/7-tmq/tmqUpdate-1ctb.py index db2ec3285dd87de532a29021142e4285c05718db..2fa6600fb99efa22035091da6d75ac00d6b55afa 100644 --- a/tests/system-test/7-tmq/tmqUpdate-1ctb.py +++ b/tests/system-test/7-tmq/tmqUpdate-1ctb.py @@ -183,6 +183,7 @@ class TDTestCase: tdLog.info("restart taosd to ensure that the data falls into the disk") tdSql.query("flush database %s"%(paraDict['dbName'])) + time.sleep(10) # update to half tables paraDict['startTs'] = paraDict['startTs'] + int(self.rowsPerTbl / 2) diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 54bfab1ebca12b28740326d624cb7e847802903c..c3ec4875ceeeb25a0ee3b9f921a8e7d5c40fdc07 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -43,9 +43,9 @@ class TDTestCase: tdSql.execute('use db_taosx') tdSql.query("show tables") if drop: - tdSql.checkRows(10) + tdSql.checkRows(11) else: - tdSql.checkRows(15) + tdSql.checkRows(16) tdSql.query("select * from jt order by i") tdSql.checkRows(2) tdSql.checkData(0, 1, 1) @@ -63,20 +63,20 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.checkData(0, 1, 1) - tdSql.checkData(1, 1, 21) + tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) - tdSql.checkData(1, 2, 21) + tdSql.checkData(2, 2, 21) tdSql.checkData(0, 5, "stt3") - tdSql.checkData(1, 5, "stt4") + tdSql.checkData(2, 5, "stt4") tdSql.execute('use abc1') tdSql.query("show tables") if drop: - tdSql.checkRows(10) + tdSql.checkRows(11) else: - tdSql.checkRows(15) + tdSql.checkRows(16) tdSql.query("select * from jt order by i") tdSql.checkRows(2) tdSql.checkData(0, 1, 1) @@ -94,13 +94,13 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(2) + tdSql.checkRows(3) tdSql.checkData(0, 1, 1) - tdSql.checkData(1, 1, 21) + tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) - tdSql.checkData(1, 2, 21) + tdSql.checkData(2, 2, 21) tdSql.checkData(0, 5, "stt3") - tdSql.checkData(1, 5, "stt4") + tdSql.checkData(2, 5, "stt4") return @@ -123,6 +123,11 @@ class TDTestCase: def checkData(self): tdSql.execute('use db_taosx') + tdSql.query("select * from tb1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 1) + tdSql.query("select * from ct3 order by c1 desc") tdSql.checkRows(2) tdSql.checkData(0, 1, 51) diff --git a/tests/system-test/pytest.sh b/tests/system-test/pytest.sh index 68d49f5d06c48d7ffcbfa564d802762b7f120b4f..792cab98f7895e00490644ccea9dbf5e8757956b 100755 --- a/tests/system-test/pytest.sh +++ b/tests/system-test/pytest.sh @@ -8,15 +8,23 @@ set +e #set -x +if [[ "$OSTYPE" == "darwin"* ]]; then + TD_OS="Darwin" +else + OS=$(cat /etc/*-release | grep "^NAME=" | cut -d= -f2) + len=$(echo ${#OS}) + len=$((len - 2)) + TD_OS=$(echo -ne ${OS:1:${len}} | cut -d" " -f1) +fi -UNAME_BIN=`which uname` -OS_TYPE=`$UNAME_BIN` +UNAME_BIN=$(which uname) +OS_TYPE=$($UNAME_BIN) cd . # Get responsible directories -CODE_DIR=`dirname $0` -CODE_DIR=`pwd` +CODE_DIR=$(dirname $0) +CODE_DIR=$(pwd) IN_TDINTERNAL="community" if [[ "$CODE_DIR" == *"$IN_TDINTERNAL"* ]]; then @@ -25,19 +33,15 @@ else cd ../../ fi -TOP_DIR=`pwd` -TAOSD_DIR=`find . -name "taosd"|grep bin|head -n1` +TOP_DIR=$(pwd) +TAOSD_DIR=$(find . -name "taosd" | grep bin | head -n1) -if [[ "$OS_TYPE" != "Darwin" ]]; then - cut_opt="--field=" -else - cut_opt="-f " -fi +cut_opt="-f " if [[ "$TAOSD_DIR" == *"$IN_TDINTERNAL"* ]]; then - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2,3` + BIN_DIR=$(find . -name "taosd" | grep bin | head -n1 | cut -d '/' ${cut_opt}2,3) else - BIN_DIR=`find . -name "taosd"|grep bin|head -n1|cut -d '/' ${cut_opt}2` + BIN_DIR=$(find . -name "taosd" | grep bin | head -n1 | cut -d '/' ${cut_opt}2) fi declare -x BUILD_DIR=$TOP_DIR/$BIN_DIR @@ -66,35 +70,38 @@ ulimit -c unlimited #sudo sysctl -w kernel.core_pattern=$TOP_DIR/core.%p.%e echo "ExcuteCmd:" $* -AsanFile=$ASAN_DIR/psim.info -echo "AsanFile:" $AsanFile - -unset LD_PRELOAD -#export LD_PRELOAD=libasan.so.5 -export LD_PRELOAD=`gcc -print-file-name=libasan.so` -echo "Preload AsanSo:" $? - -$* -a 2> $AsanFile - -unset LD_PRELOAD -for ((i=1;i<=20;i++)) -do - AsanFileLen=`cat $AsanFile | wc -l` - echo "AsanFileLen:" $AsanFileLen - if [ $AsanFileLen -gt 10 ]; then - break - fi - sleep 1 -done - -AsanFileSuccessLen=`grep -w successfully $AsanFile | wc -l` -echo "AsanFileSuccessLen:" $AsanFileSuccessLen -if [ $AsanFileSuccessLen -gt 0 ]; then - echo "Execute script successfully and check asan" - $CODE_DIR/../script/sh/checkAsan.sh +if [[ "$TD_OS" == "Alpine" ]]; then + $* else - echo "Execute script failure" - exit 1 + AsanFile=$ASAN_DIR/psim.info + echo "AsanFile:" $AsanFile + + unset LD_PRELOAD + #export LD_PRELOAD=libasan.so.5 + export LD_PRELOAD=$(gcc -print-file-name=libasan.so) + echo "Preload AsanSo:" $? + + $* -a 2>$AsanFile + + unset LD_PRELOAD + for ((i = 1; i <= 20; i++)); do + AsanFileLen=$(cat $AsanFile | wc -l) + echo "AsanFileLen:" $AsanFileLen + if [ $AsanFileLen -gt 10 ]; then + break + fi + sleep 1 + done + + AsanFileSuccessLen=$(grep -w successfully $AsanFile | wc -l) + echo "AsanFileSuccessLen:" $AsanFileSuccessLen + + if [ $AsanFileSuccessLen -gt 0 ]; then + echo "Execute script successfully and check asan" + $CODE_DIR/../script/sh/checkAsan.sh + else + echo "Execute script failure" + exit 1 + fi fi - diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index d61d25602b081d9cc6e7609f7e807add5a71e597..1461a7b373e3395ed3f4afc111a2df8ca0e5ed60 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -7,25 +7,45 @@ IF (TD_WEBSOCKET) SET(websocket_lib_file "{taosws.dll,taosws.dll.lib}") ENDIF () MESSAGE("${Green} use libtaos-ws${ColourReset}") - - include(ExternalProject) - ExternalProject_Add(taosws-rs - PREFIX "taosws-rs" - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs - BUILD_ALWAYS off - DEPENDS taos - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - PATCH_COMMAND - COMMAND git clean -f -d - BUILD_COMMAND - COMMAND cargo update - COMMAND cargo build --release -p taos-ws-sys --features native-tls-vendored - INSTALL_COMMAND - COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib - COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include - COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include - ) + IF (TD_ALPINE) + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo update + COMMAND RUSTFLAGS=-Ctarget-feature=-crt-static cargo build --release -p taos-ws-sys --features native-tls + INSTALL_COMMAND + COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ELSE() + include(ExternalProject) + ExternalProject_Add(taosws-rs + PREFIX "taosws-rs" + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/taosws-rs + BUILD_ALWAYS off + DEPENDS taos + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" + PATCH_COMMAND + COMMAND git clean -f -d + BUILD_COMMAND + COMMAND cargo update + COMMAND cargo build --release -p taos-ws-sys --features native-tls-vendored + INSTALL_COMMAND + COMMAND cp target/release/${websocket_lib_file} ${CMAKE_BINARY_DIR}/build/lib + COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/build/include + COMMAND cmake -E copy target/release/taosws.h ${CMAKE_BINARY_DIR}/build/include + ) + ENDIF () ENDIF () IF (TD_TAOS_TOOLS) diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 1e7d0ed1406d9fc251ffdb6319fede131febef78..600e33feab10fcd0b3c0ccee2de3cd61544e16d8 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,15 +26,27 @@ ELSE () SET(LINK_WEBSOCKET "") ENDIF () +IF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + ADD_DEFINITIONS(-I${CMAKE_CURRENT_SOURCE_DIR}/../../../enterprise/packaging) +ENDIF (CUS_NAME OR CUS_PROMPT OR CUS_EMAIL) + +IF (TD_LINUX AND TD_ALPINE) + SET(LINK_ARGP "/usr/lib/libargp.a") +ELSE () + SET(LINK_ARGP "") +ENDIF () + if(TD_WINDOWS) target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() - target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET} ${LINK_JEMALLOC}) + target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET} ${LINK_JEMALLOC} ${LINK_ARGP}) endif () + target_link_libraries( shell PRIVATE os common transport util ) + target_include_directories( shell PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" diff --git a/tools/shell/inc/shellAuto.h b/tools/shell/inc/shellAuto.h index 151f6da8c6555118699eedbf791da32e47e4283e..6a317fe5c93f273f5dfd78cf409d1f14a9d9f43c 100644 --- a/tools/shell/inc/shellAuto.h +++ b/tools/shell/inc/shellAuto.h @@ -16,6 +16,8 @@ #ifndef __SHELL_AUTO__ #define __SHELL_AUTO__ +#include "shellInt.h" + #define TAB_KEY 0x09 // press tab key @@ -24,13 +26,13 @@ void pressTabKey(SShellCmd* cmd); // press othr key void pressOtherKey(char c); -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit(); // set conn void shellSetConn(TAOS* conn, bool runOnce); -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit(); // callback autotab module @@ -39,7 +41,7 @@ void callbackAutoTab(char* sqlstr, TAOS* pSql, bool usedb); // introduction void printfIntroduction(); -// show all commands help +// show all commands help void showHelp(); #endif diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index e2da695c9200b201878fd8861db81c5cc493d4c5..113853bd8397fe194d0efc65a55f73f2aa865929 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -80,12 +80,13 @@ typedef struct { } SShellArgs; typedef struct { - const char* clientVersion; - const char* promptHeader; + const char *clientVersion; + char cusName[32]; + char promptHeader[32]; const char* promptContinue; const char* osname; int32_t promptSize; - char programVersion[32]; + char programVersion[256]; } SShellOsDetails; typedef struct { diff --git a/tools/shell/inc/shellTire.h b/tools/shell/inc/shellTire.h index bdcf7bcfb3310832b60cc66af5d07c16b7eb6134..e87c3ee4f33726171d107544a137bac7b649174a 100644 --- a/tools/shell/inc/shellTire.h +++ b/tools/shell/inc/shellTire.h @@ -19,7 +19,7 @@ // // The prefix search tree is a efficient storage words and search words tree, it support 95 visible ascii code character // -#define FIRST_ASCII 40 // first visiable char is '0' +#define FIRST_ASCII 40 // first visible char is '0' #define LAST_ASCII 122 // last visilbe char is 'z' // capacity save char is 95 diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 4d40de66bde2c8504369110c436275b27268ac08..19275cce8228ff280b262c87e044a770f6a6a00d 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -19,10 +19,25 @@ #include "shellInt.h" -#define TAOS_CONSOLE_PROMPT_HEADER "taos> " +#ifndef CUS_NAME + char cusName[] = "TDengine"; +#endif + +#ifndef CUS_PROMPT + char cusPrompt[] = "taos"; +#endif + +#ifndef CUS_EMAIL + char cusEmail[] = ""; +#endif + +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + #define TAOS_CONSOLE_PROMPT_CONTINUE " -> " -#define SHELL_HOST "TDengine server FQDN to connect. The default host is localhost." +#define SHELL_HOST "The server FQDN to connect. The default host is localhost." #define SHELL_PORT "The TCP/IP port number to use for the connection." #define SHELL_USER "The user name to use when connecting to the server." #define SHELL_PASSWORD "The password to use when connecting to the server." @@ -41,7 +56,6 @@ #define SHELL_PKT_LEN "Packet length used for net test, default is 1024 bytes." #define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." #define SHELL_VERSION "Print program version." -#define SHELL_EMAIL "" #ifdef WEBSOCKET #define SHELL_DSN "The dsn to use when connecting to cloud server." @@ -78,15 +92,19 @@ void shellPrintHelp() { #endif printf("%s%s%s%s\r\n", indent, "-w,", indent, SHELL_WIDTH); printf("%s%s%s%s\r\n", indent, "-V,", indent, SHELL_VERSION); - printf("\r\n\r\nReport bugs to %s.\r\n", SHELL_EMAIL); + printf("\r\n\r\nReport bugs to %s.\r\n", cusEmail); } #ifdef LINUX #include +#ifdef _ALPINE +#include +#else #include +#endif const char *argp_program_version = version; -const char *argp_program_bug_address = SHELL_EMAIL; +const char *argp_program_bug_address = cusEmail; static struct argp_option shellOptions[] = { {"host", 'h', "HOST", 0, SHELL_HOST}, @@ -388,13 +406,16 @@ static int32_t shellCheckArgs() { int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); - shell.info.clientVersion = - "Welcome to the TDengine Command Line Interface, Client Version:%s\r\n" - "Copyright (c) 2022 by TDengine, all rights reserved.\r\n\r\n"; - shell.info.promptHeader = TAOS_CONSOLE_PROMPT_HEADER; + shell.info.clientVersion = + "Welcome to the %s Command Line Interface, Client Version:%s\r\n" + "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; + strcpy(shell.info.cusName, cusName); + sprintf(shell.info.promptHeader, "%s> ", cusPrompt); shell.info.promptContinue = TAOS_CONSOLE_PROMPT_CONTINUE; - shell.info.promptSize = 6; - snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); + shell.info.promptSize = strlen(shell.info.promptHeader); + snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), + "version: %s compatible_version: %s\ngitinfo: %s\nbuildInfo: %s", version, compatible_version, gitinfo, + buildinfo); #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) shell.info.osname = "Windows"; diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 480421b5c014fae2b71bfd93ee3e4fd683b408dd..1d872e3f0d27b88eefd16c29d02c831eda6d284d 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -15,8 +15,8 @@ #define __USE_XOPEN -#include "shellInt.h" #include "shellAuto.h" +#include "shellInt.h" #include "shellTire.h" #include "tthread.h" @@ -26,14 +26,14 @@ #define UNION_ALL " union all " // extern function -void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); -void shellGetPrevCharSize(const char* str, int32_t pos, int32_t* size, int32_t* width); -void shellShowOnScreen(SShellCmd* cmd); -void shellInsertChar(SShellCmd* cmd, char* c, int size); -void shellInsertStr(SShellCmd* cmd, char* str, int size); -bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* p, int32_t len); +void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); +void shellGetPrevCharSize(const char* str, int32_t pos, int32_t* size, int32_t* width); +void shellShowOnScreen(SShellCmd* cmd); +void shellInsertChar(SShellCmd* cmd, char* c, int size); +void shellInsertStr(SShellCmd* cmd, char* str, int size); +bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* p, int32_t len); char* tireSearchWord(int type, char* pre); -bool updateTireValue(int type, bool autoFill) ; +bool updateTireValue(int type, bool autoFill); typedef struct SAutoPtr { STire* p; @@ -139,6 +139,7 @@ SWords shellCommands[] = { {"show create table \\G;", 0, 0, NULL}, {"show connections;", 0, 0, NULL}, {"show cluster;", 0, 0, NULL}, + {"show cluster alive;", 0, 0, NULL}, {"show databases;", 0, 0, NULL}, {"show dnodes;", 0, 0, NULL}, {"show dnode variables;", 0, 0, NULL}, @@ -248,7 +249,7 @@ char* db_options[] = {"keep ", "wal_retention_size ", "wal_segment_size "}; -char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "cachesize ", "wal_fsync_period ", "wal_level "}; +char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "cachesize ", "wal_fsync_period ", "wal_level "}; char* data_types[] = {"timestamp", "int", "int unsigned", "varchar(16)", @@ -270,9 +271,8 @@ char* key_systable[] = { "ins_subscriptions", "ins_streams", "ins_stream_tasks", "ins_vnodes", "ins_user_privileges", "perf_connections", "perf_queries", "perf_consumers", "perf_trans", "perf_apps"}; - // -// ------- gobal variant define --------- +// ------- global variant define --------- // int32_t firstMatchIndex = -1; // first match shellCommands index int32_t lastMatchIndex = -1; // last match shellCommands index @@ -338,23 +338,30 @@ int cntDel = 0; // delete byte count after next press tab // show auto tab introduction void printfIntroduction() { - printf(" ****************************** Tab Completion *************************************\n"); - printf(" * The TDengine CLI supports tab completion for a variety of items, *\n"); - printf(" * including database names, table names, function names and keywords. *\n"); - printf(" * The full list of shortcut keys is as follows: *\n"); - printf(" * [ TAB ] ...... complete the current word *\n"); - printf(" * ...... if used on a blank line, display all supported commands *\n"); - printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); - printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); - printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); - printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); - printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); - printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); - printf(" *************************************************************************************\n\n"); + printf(" ****************************** Tab Completion **********************************\n"); + char secondLine[160] = "\0"; + sprintf(secondLine, " * The %s CLI supports tab completion for a variety of items, ", shell.info.cusName); + printf("%s", secondLine); + int secondLineLen = strlen(secondLine); + while (84 - (secondLineLen++) > 0) { + printf(" "); + } + printf("*\n"); + printf(" * including database names, table names, function names and keywords. *\n"); + printf(" * The full list of shortcut keys is as follows: *\n"); + printf(" * [ TAB ] ...... complete the current word *\n"); + printf(" * ...... if used on a blank line, display all supported commands *\n"); + printf(" * [ Ctrl + A ] ...... move cursor to the st[A]rt of the line *\n"); + printf(" * [ Ctrl + E ] ...... move cursor to the [E]nd of the line *\n"); + printf(" * [ Ctrl + W ] ...... move cursor to the middle of the line *\n"); + printf(" * [ Ctrl + L ] ...... clear the entire screen *\n"); + printf(" * [ Ctrl + K ] ...... clear the screen after the cursor *\n"); + printf(" * [ Ctrl + U ] ...... clear the screen before the cursor *\n"); + printf(" **************************************************************************************\n\n"); } void showHelp() { - printf("\nThe TDengine CLI supports the following commands:"); + printf("\nThe %s CLI supports the following commands:", shell.info.cusName); printf( "\n\ ----- A ----- \n\ @@ -438,6 +445,7 @@ void showHelp() { show create table ;\n\ show connections;\n\ show cluster;\n\ + show cluster alive;\n\ show databases;\n\ show dnodes;\n\ show dnode variables;\n\ @@ -606,7 +614,7 @@ void GenerateVarType(int type, char** p, int count) { // -------------------- shell auto ---------------- // -// init shell auto funciton , shell start call once +// init shell auto function , shell start call once bool shellAutoInit() { // command int32_t count = SHELL_COMMAND_COUNT(); @@ -645,7 +653,7 @@ void shellSetConn(TAOS* conn, bool runOnce) { if (!runOnce) updateTireValue(WT_VAR_DBNAME, false); } -// exit shell auto funciton, shell exit call once +// exit shell auto function, shell exit call once void shellAutoExit() { // free command int32_t count = SHELL_COMMAND_COUNT(); @@ -662,7 +670,7 @@ void shellAutoExit() { } } taosThreadMutexUnlock(&tiresMutex); - // destory + // destroy taosThreadMutexDestroy(&tiresMutex); // free threads @@ -683,7 +691,7 @@ void shellAutoExit() { // // ------------------- auto ptr for tires -------------------------- // -bool setNewAuotPtr(int type, STire* pNew) { +bool setNewAutoPtr(int type, STire* pNew) { if (pNew == NULL) return false; taosThreadMutexLock(&tiresMutex); @@ -726,13 +734,13 @@ void putBackAutoPtr(int type, STire* tire) { if (tires[type] != tire) { // update by out, can't put back , so free if (--tire->ref == 1) { - // support multi thread getAuotPtr + // support multi thread getAutoPtr freeTire(tire); } } else { tires[type]->ref--; - assert(tires[type]->ref > 0); + ASSERT(tires[type]->ref > 0); } taosThreadMutexUnlock(&tiresMutex); @@ -781,14 +789,14 @@ int writeVarNames(int type, TAOS_RES* tres) { } while (row != NULL); // replace old tire - setNewAuotPtr(type, tire); + setNewAutoPtr(type, tire); return numOfRows; } void setThreadNull(int type) { taosThreadMutexLock(&tiresMutex); - if(threads[type]) { + if (threads[type]) { taosMemoryFree(threads[type]); } threads[type] = NULL; @@ -865,7 +873,7 @@ bool updateTireValue(int type, bool autoFill) { // only match next one word from all match words, return valuue must free by caller char* matchNextPrefix(STire* tire, char* pre) { SMatch* match = NULL; - if(tire == NULL) return NULL; + if (tire == NULL) return NULL; // re-use last result if (lastMatch) { @@ -903,7 +911,7 @@ char* matchNextPrefix(STire* tire, char* pre) { if (cursorVar == -1) { // first cursorVar = 0; - return strdup(match->head->word); + return taosStrdup(match->head->word); } // according to cursorVar , calculate next one @@ -919,7 +927,7 @@ char* matchNextPrefix(STire* tire, char* pre) { cursorVar = i; } - return strdup(item->word); + return taosStrdup(item->word); } // check end item @@ -927,7 +935,7 @@ char* matchNextPrefix(STire* tire, char* pre) { // if cursorVar > var list count, return last and reset cursorVar cursorVar = -1; - return strdup(item->word); + return taosStrdup(item->word); } // move next @@ -951,7 +959,7 @@ char* tireSearchWord(int type, char* pre) { return matchNextPrefix(tire, pre); } - if(updateTireValue(type, true)) { + if (updateTireValue(type, true)) { return NULL; } @@ -1070,7 +1078,7 @@ SWords* matchCommand(SWords* input, bool continueSearch) { for (int32_t i = 0; i < count; i++) { SWords* shellCommand = shellCommands + i; if (continueSearch && lastMatchIndex != -1 && i <= lastMatchIndex) { - // new match must greate than lastMatchIndex + // new match must greater than lastMatchIndex if (varMode && i == lastMatchIndex) { // do nothing, var match on lastMatchIndex } else { @@ -1156,7 +1164,7 @@ void printScreen(TAOS* con, SShellCmd* cmd, SWords* match) { // main key press tab , matched return true else false bool firstMatchCommand(TAOS* con, SShellCmd* cmd) { - if(con == NULL || cmd == NULL) return false; + if (con == NULL || cmd == NULL) return false; // parse command SWords* input = (SWords*)taosMemoryMalloc(sizeof(SWords)); memset(input, 0, sizeof(SWords)); @@ -1200,7 +1208,7 @@ void createInputFromFirst(SWords* input, SWords* firstMatch) { for (int i = 0; i < firstMatch->matchIndex && word; i++) { // combine source from each word strncpy(input->source + input->source_len, word->word, word->len); - strcat(input->source, " "); // append blank splite + strcat(input->source, " "); // append blank space input->source_len += word->len + 1; // 1 is blank length // move next word = word->next; @@ -1429,7 +1437,7 @@ bool appendAfterSelect(TAOS* con, SShellCmd* cmd, char* sql, int32_t len) { return true; } - // fill funciton + // fill function if (fieldEnd) { // fields is end , need match keyword ret = fillWithType(con, cmd, last, WT_VAR_KEYWORD); @@ -1527,9 +1535,9 @@ bool matchSelectQuery(TAOS* con, SShellCmd* cmd) { // if is input create fields or tags area, return true bool isCreateFieldsArea(char* p) { - // put to while, support like create table st(ts timestamp, bin1 binary(16), bin2 + blank + TAB - char* p1 = strdup(p); - bool ret = false; + // put to while, support like create table st(ts timestamp, bin1 binary(16), bin2 + blank + TAB + char* p1 = taosStrdup(p); + bool ret = false; while (1) { char* left = strrchr(p1, '('); if (left == NULL) { @@ -1550,7 +1558,7 @@ bool isCreateFieldsArea(char* p) { ret = true; break; } - + // set string end by small for next strrchr search *left = 0; } @@ -1612,7 +1620,7 @@ bool matchCreateTable(TAOS* con, SShellCmd* cmd) { // tb options if (!ret) { - // find like create talbe st (...) tags(..) + // find like create table st (...) tags(..) char* p1 = strchr(ps, ')'); // first ')' end if (p1) { if (strchr(p1 + 1, ')')) { // second ')' end @@ -1704,21 +1712,21 @@ bool matchOther(TAOS* con, SShellCmd* cmd) { // last match if nothing matched bool matchEnd(TAOS* con, SShellCmd* cmd) { // str dump - bool ret = false; + bool ret = false; char* ps = strndup(cmd->command, cmd->commandSize); char* last = lastWord(ps); - char* elast = strrchr(last, '.'); // find end last - if(elast) { + char* elast = strrchr(last, '.'); // find end last + if (elast) { last = elast + 1; } // less one char can match - if(strlen(last) == 0 ) { + if (strlen(last) == 0) { goto _return; } // match database - if(elast == NULL) { + if (elast == NULL) { // dot need not completed with dbname if (fillWithType(con, cmd, last, WT_VAR_DBNAME)) { ret = true; @@ -1741,7 +1749,7 @@ void pressTabKey(SShellCmd* cmd) { // check empty tab key if (cmd->commandSize == 0) { // have multi line tab key - if(cmd->bufferSize == 0) { + if (cmd->bufferSize == 0) { showHelp(); } shellShowOnScreen(cmd); diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 5235030cf91cac5d41b782544da9d8a7be4e1349..0e305f57e9c781a8149db1fecccd06a703bc1465 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -40,7 +40,7 @@ static void shellPositionCursorEnd(SShellCmd *cmd); static void shellPrintChar(char c, int32_t times); static void shellPositionCursor(int32_t step, int32_t direction); static void shellUpdateBuffer(SShellCmd *cmd); -static int32_t shellIsReadyGo(SShellCmd *cmd); +static bool shellIsReadyGo(SShellCmd *cmd); static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width); static void shellResetCommand(SShellCmd *cmd, const char s[]); void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); @@ -62,7 +62,8 @@ int32_t shellCountPrefixOnes(uint8_t c) { } void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos > 0); + ASSERT(pos > 0); + if (pos <= 0) return; TdWchar wc; *size = 0; @@ -75,13 +76,14 @@ void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - // assert(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk + // ASSERT(rc == *size); // it will be core, if str is encode by utf8 and taos charset is gbk *width = taosWcharWidth(wc); } void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { - assert(pos >= 0); + ASSERT(pos >= 0); + if(pos < 0) return; TdWchar wc; *size = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); @@ -89,7 +91,8 @@ void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t * } void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; TdWchar wc; if (taosMbToWchar(&wc, c, size) < 0) return; @@ -135,7 +138,8 @@ void shellInsertStr(SShellCmd *cmd, char *str, int32_t size) { } void shellBackspaceChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -155,7 +159,8 @@ void shellBackspaceChar(SShellCmd *cmd) { } void shellClearLineBefore(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); @@ -169,7 +174,8 @@ void shellClearLineBefore(SShellCmd *cmd) { } void shellClearLineAfter(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->commandSize -= cmd->endOffset - cmd->cursorOffset; @@ -178,7 +184,8 @@ void shellClearLineAfter(SShellCmd *cmd) { } void shellDeleteChar(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -196,7 +203,8 @@ void shellDeleteChar(SShellCmd *cmd) { } void shellMoveCursorLeft(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -210,7 +218,8 @@ void shellMoveCursorLeft(SShellCmd *cmd) { } void shellMoveCursorRight(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -224,7 +233,8 @@ void shellMoveCursorRight(SShellCmd *cmd) { } void shellPositionCursorHome(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset > 0) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -244,7 +254,8 @@ void positionCursorMiddle(SShellCmd *cmd) { } void shellPositionCursorEnd(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (cmd->cursorOffset < cmd->commandSize) { shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); @@ -279,7 +290,8 @@ void shellPositionCursor(int32_t step, int32_t direction) { } void shellUpdateBuffer(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); strcat(cmd->buffer, cmd->command); @@ -293,8 +305,9 @@ void shellUpdateBuffer(SShellCmd *cmd) { shellShowOnScreen(cmd); } -int32_t shellIsReadyGo(SShellCmd *cmd) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); +bool shellIsReadyGo(SShellCmd *cmd) { + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return false; char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize); @@ -305,11 +318,11 @@ int32_t shellIsReadyGo(SShellCmd *cmd) { "\\s*clear\\s*$)"; if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); - return 1; + return true; } taosMemoryFree(total); - return 0; + return false; } void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { @@ -321,7 +334,8 @@ void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { } void shellResetCommand(SShellCmd *cmd, const char s[]) { - assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + ASSERT(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); + if(cmd->cursorOffset > cmd->commandSize || cmd->endOffset < cmd->screenOffset) return; shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE); @@ -399,7 +413,7 @@ void shellShowOnScreen(SShellCmd *cmd) { int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); if (ret < 0) break; size += ret; - /* assert(size >= 0); */ + /* ASSERT(size >= 0); */ int32_t width = taosWcharWidth(wc); if (remain_column > width) { printf("%lc", wc); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 9b842a9e6f0ba38e90804ff853bc9cad07cfcd52..655fef383e5f6eee40b9b42d4cf59ecf3608d71d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -127,7 +127,7 @@ void shellRecordCommandToHistory(char *command) { if (pHistory->hist[pHistory->hend] != NULL) { taosMemoryFreeClear(pHistory->hist[pHistory->hend]); } - pHistory->hist[pHistory->hend] = strdup(command); + pHistory->hist[pHistory->hend] = taosStrdup(command); pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; if (pHistory->hend == pHistory->hstart) { @@ -713,7 +713,7 @@ int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { } default: - assert(false); + ASSERT(false); } return 0; @@ -821,7 +821,7 @@ void shellReadHistory() { while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { line[read_size - 1] = '\0'; taosMemoryFree(pHistory->hist[pHistory->hend]); - pHistory->hist[pHistory->hend] = strdup(line); + pHistory->hist[pHistory->hend] = taosStrdup(line); pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; @@ -845,6 +845,8 @@ void shellReadHistory() { i = (i + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; } taosFprintfFile(pFile, "%s\n", pHistory->hist[endIndex]); + + /* coverity[+retval] */ taosFsyncFile(pFile); taosCloseFile(&pFile); } @@ -1072,7 +1074,8 @@ void *shellThreadLoop(void *arg) { } int32_t shellExecute() { - printf(shell.info.clientVersion, taos_get_client_info()); + printf(shell.info.clientVersion, shell.info.cusName, + taos_get_client_info(), shell.info.cusName); fflush(stdout); SShellArgs *pArgs = &shell.args; @@ -1104,7 +1107,7 @@ int32_t shellExecute() { if (runOnce) { if (pArgs->commands != NULL) { printf("%s%s\r\n", shell.info.promptHeader, pArgs->commands); - char *cmd = strdup(pArgs->commands); + char *cmd = taosStrdup(pArgs->commands); shellRunCommand(cmd, true); taosMemoryFree(cmd); } @@ -1128,7 +1131,7 @@ int32_t shellExecute() { } if (tsem_init(&shell.cancelSem, 0, 0) != 0) { - printf("failed to create cancel semphore\r\n"); + printf("failed to create cancel semaphore\r\n"); return -1; } diff --git a/tools/shell/src/shellTire.c b/tools/shell/src/shellTire.c index 0ce0588cce8c143693576adaa08ea3c329a6d45c..a8726f312652734cbff34da1f4a2af0307b2822f 100644 --- a/tools/shell/src/shellTire.c +++ b/tools/shell/src/shellTire.c @@ -75,7 +75,7 @@ void freeTire(STire* tire) { // insert a new word to list bool insertToList(STire* tire, char* word) { StrName* p = (StrName*)taosMemoryMalloc(sizeof(StrName)); - p->name = strdup(word); + p->name = taosStrdup(word); p->next = NULL; if (tire->head == NULL) { @@ -218,7 +218,7 @@ void addWordToMatch(SMatch* match, char* word) { // malloc new SMatchNode* node = (SMatchNode*)taosMemoryMalloc(sizeof(SMatchNode)); memset(node, 0, sizeof(SMatchNode)); - node->word = strdup(word); + node->word = taosStrdup(word); // append to match if (match->head == NULL) { diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 8c47d165557317dbcf710dda7d72b77037340692..e15b49efcc35da2682d003243c0a19eb278acbc7 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -50,19 +50,19 @@ bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) { int32_t shellCheckIntSize() { if (sizeof(int8_t) != 1) { - printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); + printf("int8 size is %d(!= 1)", (int)sizeof(int8_t)); return -1; } if (sizeof(int16_t) != 2) { - printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); + printf("int16 size is %d(!= 2)", (int)sizeof(int16_t)); return -1; } if (sizeof(int32_t) != 4) { - printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); + printf("int32 size is %d(!= 4)", (int)sizeof(int32_t)); return -1; } if (sizeof(int64_t) != 8) { - printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + printf("int64 size is %d(!= 8)", (int)sizeof(int64_t)); return -1; } return 0; @@ -80,7 +80,7 @@ void shellGenerateAuth() { void shellDumpConfig() { SConfig *pCfg = taosGetCfg(); if (pCfg == NULL) { - printf("TDengine read global config failed!\r\n"); + printf("read global config failed!\r\n"); } else { cfgDumpCfg(pCfg, 1, true); } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index e848c07b666fb24cb876b4637e0fe7e3a3c44f4d..81c23035e21a225f98e8214bd5c3adcb25add72c 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -37,7 +37,9 @@ static int horizontalPrintWebsocket(WS_RES* wres, double* execute_time) { const void* data = NULL; int rows; ws_fetch_block(wres, &data, &rows); - *execute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *execute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -77,7 +79,9 @@ static int verticalPrintWebsocket(WS_RES* wres, double* pexecute_time) { int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { return 0; } @@ -129,7 +133,9 @@ static int dumpWebsocketToFile(const char* fname, WS_RES* wres, double* pexecute int rows = 0; const void* data = NULL; ws_fetch_block(wres, &data, &rows); - *pexecute_time += (double)(ws_take_timing(wres)/1E6); + if (wres) { + *pexecute_time += (double)(ws_take_timing(wres)/1E6); + } if (!rows) { taosCloseFile(&pFile); return 0; @@ -228,17 +234,23 @@ void shellRunSingleCommandWebsocketImp(char *command) { if (code == TSDB_CODE_WS_SEND_TIMEOUT || code == TSDB_CODE_WS_RECV_TIMEOUT) { fprintf(stderr, "Hint: use -t to increase the timeout in seconds\n"); } else if (code == TSDB_CODE_WS_INTERNAL_ERRO || code == TSDB_CODE_WS_CLOSED) { - fprintf(stderr, "TDengine server is disconnected, will try to reconnect\n"); shell.ws_conn = NULL; } ws_free_result(res); - if (reconnectNum == 0) continue; + if (reconnectNum == 0) { + continue; + } else { + fprintf(stderr, "The server is disconnected, will try to reconnect\n"); + } return; } break; } - double execute_time = ws_take_timing(res)/1E6; + double execute_time = 0; + if (res) { + execute_time = ws_take_timing(res)/1E6; + } if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\r\n\r\n"); diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index b048b79e9b71058faaccdadeb723c1e0f46dce58..6ca266c55533b644c47ad61eb3bebc273ae7be7f 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(sml_test sml_test.c) +add_executable(get_db_name_test get_db_name_test.c) target_link_libraries( create_table PUBLIC taos_static @@ -40,3 +41,11 @@ target_link_libraries( PUBLIC common PUBLIC os ) + +target_link_libraries( + get_db_name_test + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) diff --git a/utils/test/c/get_db_name_test.c b/utils/test/c/get_db_name_test.c new file mode 100644 index 0000000000000000000000000000000000000000..ebbfdc84a77a0eb3d76a5b561c9b280b3930c7a6 --- /dev/null +++ b/utils/test/c/get_db_name_test.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" +#include "types.h" +#include "tlog.h" + +int get_db_test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db vgroups 2"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + int code = taos_errno(pRes); + taos_free_result(pRes); + ASSERT(code == 0); + + code = taos_get_current_db(taos, NULL, 0, NULL); + ASSERT(code != 0); + + int required = 0; + code = taos_get_current_db(taos, NULL, 0, &required); + ASSERT(code != 0); + ASSERT(required == 7); + + char database[10] = {0}; + code = taos_get_current_db(taos, database, 3, &required); + ASSERT(code != 0); + ASSERT(required == 7); + ASSERT(strcpy(database, "sm")); + + char database1[10] = {0}; + code = taos_get_current_db(taos, database1, 10, &required); + ASSERT(code == 0); + ASSERT(strcpy(database1, "sml_db")); + + taos_close(taos); + + return code; +} + +int main(int argc, char *argv[]) { + int ret = 0; + ret = get_db_test(); + ASSERT(!ret); + return ret; +} diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index c6073541fdfadacebc4ef49e8fd00cd0a5ef0fbb..c36ab388778a80f4dd9349e0cb04551166007bee 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -19,8 +19,8 @@ #include #include #include "taos.h" -#include "types.h" #include "tlog.h" +#include "types.h" int smlProcess_influx_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -78,12 +78,23 @@ int smlProcess_telnet_Test() { pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - const char *sql[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", - "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", - "sys.if.bytes.out 1479496102 1.3E3 network=tcp", - " sys.procs.running 1479496100 42 host=web01 "}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // char *sql[4] = {0}; + // sql[0] = taosMemoryCalloc(1, 128); + // sql[1] = taosMemoryCalloc(1, 128); + // sql[2] = taosMemoryCalloc(1, 128); + // sql[3] = taosMemoryCalloc(1, 128); + const char *sql1[] = {"sys.if.bytes.out 1479496100 1.3E0 host=web01 interface=eth0", + "sys.if.bytes.out 1479496101 1.3E1 interface=eth0 host=web01 ", + "sys.if.bytes.out 1479496102 1.3E3 network=tcp", + " sys.procs.running 1479496100 42 host=web01 "}; + + // for(int i = 0; i < 4; i++){ + // strncpy(sql[i], sql1[i], 128); + // } + + // pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_TELNET_PROTOCOL, + // TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_TELNET_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -103,41 +114,62 @@ int smlProcess_json1_Test() { taos_free_result(pRes); const char *sql[] = { - "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}}]" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":0,\"value\":18,\"tags\":{\"host\":\"web01\",\"id\":\"t1\",\"dc\":" + "\"lga\"}},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344045,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":" + "\"lga\"}}]"}; + + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); - taos_close(taos); - return code; -} - -int smlProcess_json2_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); + const char *sql2[] = { + "[{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344041,\"value\":13,\"tags\":{\"host\":\"web01\",\"dc\":\"lga\"}" + "},{\"metric\":\"sys.cpu.nice\",\"timestamp\":1662344042,\"value\":9,\"tags\":{\"host\":\"web02\",\"dc\":\"lga\"}" + "}]", + }; - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); + char *sql3[1] = {0}; + for (int i = 0; i < 1; i++) { + sql3[i] = taosMemoryCalloc(1, 1024); + strncpy(sql3[i], sql2[i], 1023); + } - const char *sql[] = { - "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\",\"type\":\"binary\"},\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + pRes = taos_schemaless_insert(taos, (char **)sql3, sizeof(sql3) / sizeof(sql3[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); + code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); + + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql3[i]); + } + taos_close(taos); return code; } -int smlProcess_json3_Test() { +int smlProcess_json2_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -147,18 +179,33 @@ int smlProcess_json3_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current1\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3,\"type\":\"i64\"},\"tags\":{\"t1\":{\"value\":2,\"type\":\"bigint\"},\"t2\":{\"value\":2,\"type\":\"int\"},\"t3\":{\"value\":2,\"type\":\"i16\"},\"t4\":{\"value\":2,\"type\":\"i8\"},\"t5\":{\"value\":2,\"type\":\"f32\"},\"t6\":{\"value\":2,\"type\":\"double\"},\"t7\":{\"value\":\"8323\",\"type\":\"binary\"},\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":{\"value\":true,\"type\":\"bool\"},\"id\":\"d1001\"}}"}; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "{\"metric\":\"meter_current0\",\"timestamp\":{\"value\":1662344042,\"type\":\"s\"},\"value\":{\"value\":10.3," + "\"type\":\"i64\"},\"tags\":{\"groupid\":{\"value\":2,\"type\":\"bigint\"},\"location\":{\"value\":\"北京\"," + "\"type\":\"binary\"},\"id\":\"d1001\"}}"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } -int smlProcess_json4_Test() { +int smlProcess_json3_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); @@ -168,15 +215,28 @@ int smlProcess_json4_Test() { taos_free_result(pRes); const char *sql[] = { - "{\"metric\":\"meter_current2\",\"timestamp\":{\"value\":1662344042000,\"type\":\"ms\"},\"value\":\"ni\",\"tags\":{\"t1\":{\"value\":20,\"type\":\"i64\"},\"t2\":{\"value\":25,\"type\":\"i32\"},\"t3\":{\"value\":2,\"type\":\"smallint\"},\"t4\":{\"value\":2,\"type\":\"tinyint\"},\"t5\":{\"value\":2,\"type\":\"float\"},\"t6\":{\"value\":0.2,\"type\":\"f64\"},\"t7\":\"nsj\",\"t8\":{\"value\":\"北京\",\"type\":\"nchar\"},\"t9\":false,\"id\":\"d1001\"}}" - }; - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, + "[{\"metric\":\"sys.cpu.nice3\",\"timestamp\":0,\"value\":\"18\",\"tags\":{\"host\":\"web01\",\"id\":\"t1\"," + "\"dc\":\"lga\"}}]"}; + char *sql1[1] = {0}; + for (int i = 0; i < 1; i++) { + sql1[i] = taosMemoryCalloc(1, 1024); + strncpy(sql1[i], sql[i], 1023); + } + + pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); + if (code != 0) { + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + } else { + printf("%s result:success\n", __FUNCTION__); + } taos_free_result(pRes); taos_close(taos); + for (int i = 0; i < 1; i++) { + taosMemoryFree(sql1[i]); + } return code; } @@ -242,10 +302,10 @@ int sml_16384_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); - if(code) return code; + if (code) return code; const char *sql1[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c0=f,c1=127i8,c11=L\"ncharColValue\",c10=t 1626006833631000000", }; pRes = taos_schemaless_insert(taos, (char **)sql1, 1, TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -677,52 +737,6 @@ int sml_oom_Test() { return code; } -int sml_16368_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639000, \"type\": \"us\"}, \"value\": 1, " - "\"tags\": {\"t1\": 3, \"t2\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t3\", \"type\": " - "\"binary\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833739000, \"type\": \"us\"}, \"value\": 2, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639100, \"type\": \"us\"}, \"value\": 3, " - "\"tags\": {\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t3\": {\"value\": \"ste\", \"type\": \"nchar\"}}}," - "{\"metric\": \"stf567890\", \"timestamp\": {\"value\": 1626006833639200, \"type\": \"us\"}, \"value\": 4, " - "\"tags\": {\"t1\": {\"value\": 4, \"type\": \"bigint\"}, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833639300, \"type\": \"us\"}, \"value\": " - "{\"value\": 5, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t2\": 5.0, " - "\"t3\": {\"value\": \"t4\", \"type\": \"binary\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006833639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 6, \"type\": \"double\"}, \"tags\": {\"t2\": 5.0, \"t3\": {\"value\": \"ste2\", \"type\": " - "\"nchar\"}}}," - "{\"metric\": \"stb_name\", \"timestamp\": {\"value\": 1626006834639400, \"type\": \"us\"}, \"value\": " - "{\"value\": 7, \"type\": \"double\"}, \"tags\": {\"t2\": {\"value\": 5.0, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"ste2\", \"type\": \"nchar\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833839006, \"type\": \"us\"}, \"value\": " - "{\"value\": 8, \"type\": \"double\"}, \"tags\": {\"t1\": {\"value\": 4, \"type\": \"double\"}, \"t3\": " - "{\"value\": \"t4\", \"type\": \"binary\"}, \"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, " - "\"type\": \"double\"}}}," - "{\"metric\": \"st123456\", \"timestamp\": {\"value\": 1626006833939007, \"type\": \"us\"}, \"value\": " - "{\"value\": 9, \"type\": \"double\"}, \"tags\": {\"t1\": 4, \"t3\": {\"value\": \"t4\", \"type\": \"binary\"}, " - "\"t2\": {\"value\": 5, \"type\": \"double\"}, \"t4\": {\"value\": 5, \"type\": \"double\"}}}]"}; - pRes = taos_schemaless_insert(taos, (char **)sql, 0, TSDB_SML_JSON_PROTOCOL, TSDB_SML_TIMESTAMP_MICRO_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_dup_time_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -732,7 +746,7 @@ int sml_dup_time_Test() { const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " @@ -762,214 +776,6 @@ int sml_dup_time_Test() { return code; } -int sml_16960_Test() { - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); - taos_free_result(pRes); - - pRes = taos_query(taos, "use sml_db"); - taos_free_result(pRes); - - const char *sql[] = { - "[" - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955000, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": 830525384, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955001, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -588348364, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955002, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -370310823, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955003, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -811250191, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}," - "{" - "\"timestamp\":" - "" - "{ \"value\": 1664418955004, \"type\": \"ms\" }" - "," - "\"value\":" - "" - "{ \"value\": -330340558, \"type\": \"int\" }" - "," - "\"tags\": {" - "\"id\": \"stb00_0\"," - "\"t0\":" - "" - "{ \"value\": 83972721, \"type\": \"int\" }" - "," - "\"t1\":" - "" - "{ \"value\": 539147525, \"type\": \"int\" }" - "," - "\"t2\":" - "" - "{ \"value\": 618258572, \"type\": \"int\" }" - "," - "\"t3\":" - "" - "{ \"value\": -10536201, \"type\": \"int\" }" - "," - "\"t4\":" - "" - "{ \"value\": 349227409, \"type\": \"int\" }" - "," - "\"t5\":" - "" - "{ \"value\": 249347042, \"type\": \"int\" }" - "}," - "\"metric\": \"stb0\"" - "}" - "]"}; - - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_JSON_PROTOCOL, - TSDB_SML_TIMESTAMP_MILLI_SECONDS); - printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); - int code = taos_errno(pRes); - taos_free_result(pRes); - taos_close(taos); - - return code; -} - int sml_add_tag_col_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -977,8 +783,10 @@ int sml_add_tag_col_Test() { taos_free_result(pRes); const char *sql[] = { - "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" - }; + "macylr,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64," + "t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=" + "\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -989,8 +797,10 @@ int sml_add_tag_col_Test() { if (code) return code; const char *sql1[] = { - "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000" - }; + "macylr,id=macylr_17875_1804,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=" + "22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\",t11=127i8,t10=L\"ncharTagValue\" " + "c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c8=" + "L\"ncharColValue\",c9=7u64,c11=L\"ncharColValue\",c10=f 1626006833639000000"}; pRes = taos_schemaless_insert(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, 0); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1004,10 +814,10 @@ int sml_add_tag_col_Test() { int smlProcess_18784_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists db_18784 schemaless 1"); taos_free_result(pRes); - pRes = taos_query(taos, "use sml_db"); + pRes = taos_query(taos, "use db_18784"); taos_free_result(pRes); const char *sql[] = { @@ -1018,7 +828,7 @@ int smlProcess_18784_Test() { printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); int code = taos_errno(pRes); ASSERT(!code); - ASSERT(taos_affected_rows(pRes) == 2); + ASSERT(taos_affected_rows(pRes) == 1); taos_free_result(pRes); pRes = taos_query(taos, "select * from disk"); @@ -1027,26 +837,26 @@ int smlProcess_18784_Test() { ASSERT(fieldNum == 5); printf("fieldNum:%d\n", fieldNum); TAOS_ROW row = NULL; - int32_t rowIndex = 0; - while((row = taos_fetch_row(pRes)) != NULL) { - int64_t ts = *(int64_t*)row[0]; - int64_t used = *(int64_t*)row[1]; - int64_t total = *(int64_t*)row[2]; - int64_t freed = *(int64_t*)row[3]; - if(rowIndex == 0){ + int32_t rowIndex = 0; + while ((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t *)row[0]; + int64_t used = *(int64_t *)row[1]; + int64_t total = *(int64_t *)row[2]; + int64_t freed = *(int64_t *)row[3]; + if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); ASSERT(total == 1081101176832); ASSERT(freed == 66932805); -// ASSERT_EQ(latitude, 24.5208); -// ASSERT_EQ(longitude, 28.09377); -// ASSERT_EQ(elevation, 428); -// ASSERT_EQ(velocity, 0); -// ASSERT_EQ(heading, 304); -// ASSERT_EQ(grade, 0); -// ASSERT_EQ(fuel_consumption, 25); - }else{ -// ASSERT(0); + // ASSERT_EQ(latitude, 24.5208); + // ASSERT_EQ(longitude, 28.09377); + // ASSERT_EQ(elevation, 428); + // ASSERT_EQ(velocity, 0); + // ASSERT_EQ(heading, 304); + // ASSERT_EQ(grade, 0); + // ASSERT_EQ(fuel_consumption, 25); + } else { + // ASSERT(0); } rowIndex++; } @@ -1063,17 +873,21 @@ int sml_19221_Test() { taos_free_result(pRes); const char *sql[] = { - "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 1626006833639000000", + "qelhxo,id=pnnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833632000000\nqelhxo,id=pnnhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833633000000\n#comment\nqelhxo,id=pnqhsa,t0=t,t1=127i8 c11=L\"ncharColValue\",c0=t,c1=127i8 " + "1626006833634000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - char* tmp = (char*)taosMemoryCalloc(1024, 1); + char *tmp = (char *)taosMemoryCalloc(1024, 1); memcpy(tmp, sql[0], strlen(sql[0])); - *(char*)(tmp+44) = 0; + *(char *)(tmp + 44) = 0; int32_t totalRows = 0; - pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert_raw(taos, tmp, strlen(sql[0]), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); ASSERT(totalRows == 3); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); @@ -1088,19 +902,22 @@ int sml_19221_Test() { int sml_ts2164_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_test BUFFER 384 MINROWS 1000 PAGES 256 PRECISION 'ns'"); taos_free_result(pRes); const char *sql[] = { + // "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", + "meters,location=la,groupid=ca current=11.8,voltage=221", "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=ca current=11.8,voltage=221,phase=0.27", - "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", + // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; pRes = taos_query(taos, "use line_test"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_MILLI_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1113,26 +930,42 @@ int sml_ts2164_Test() { int sml_ttl_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "drop database if exists sml_db"); + taos_free_result(pRes); + + pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); taos_free_result(pRes); const char *sql[] = { - "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" 1626006833739000000", + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833739000000", + }; + const char *sql1[] = { + "meters,location=California.LosAngeles,groupid=2 current=11.8,voltage=221,phase=\"2022-02-0210:22:22\" " + "1626006833339000000", }; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); - pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); taos_free_result(pRes); - pRes = taos_query(taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); + pRes = taos_schemaless_insert_ttl(taos, (char **)sql1, sizeof(sql1) / sizeof(sql1[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS, 20); + + printf("%s result1:%s\n", __FUNCTION__, taos_errstr(pRes)); + taos_free_result(pRes); + + pRes = taos_query( + taos, "select `ttl` from information_schema.ins_tables where table_name='t_be97833a0e1f523fcdaeb6291d6fdf27'"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); TAOS_ROW row = taos_fetch_row(pRes); - if(row != NULL && row[0] != NULL){ - int32_t ttl = *(int32_t*)row[0]; + if (row != NULL && *row != NULL) { + int32_t ttl = *(int32_t *)row[0]; ASSERT(ttl == 20); } @@ -1143,23 +976,91 @@ int sml_ttl_Test() { return code; } +// char *str[] ={ +// "", +// "f64", +// "F64", +// "f32", +// "F32", +// "i", +// "I", +// "i64", +// "I64", +// "u", +// "U", +// "u64", +// "U64", +// "i32", +// "I32", +// "u32", +// "U32", +// "i16", +// "I16", +// "u16", +// "U16", +// "i8", +// "I8", +// "u8", +// "U8", +// }; +// uint8_t smlCalTypeSum(char* endptr, int32_t left){ +// uint8_t sum = 0; +// for(int i = 0; i < left; i++){ +// sum += endptr[i]; +// } +// return sum; +// } + int sml_ts2385_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS_RES *pRes = taos_query(taos, "CREATE DATABASE IF NOT EXISTS ts2385"); taos_free_result(pRes); - const char *sql[] ={ - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006,m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true,m14=0,s14=false,s16=true,k5=0,hex=\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0,s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525,s4=false,s6=false 1672818929178749400", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0,k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=0,k16=0 1672818919126971000", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0,s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=false,s19=false,hex=\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false,k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0,k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", - "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", - "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false,s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=false,k2=0,k10=0,hex=\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0,k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800" - }; + const char *sql[] = { + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "s5=false,s18=false,k14=0,k2=0,k8=0,k10=0,s9=false,s19=false,k11=0,k13=0,s22=false,k15=0,m2=37.416671660000006," + "m8=600,m10=1532,m1=20.25,m13=0,s7=false,k7=0,m16=0,s17=false,k4=0,s11=false,s15=true,m7=600,m12=1490,s1=true," + "m14=0,s14=false,s16=true,k5=0,hex=" + "\"7b3b00000001030301030200000000323231313233304339344b30002b01012a10028003000000070d05da025802580258025802580258" + "045305fc05f505d200000000000000000afc7d\",k6=0,m3=600,s3=false,s24=false,k3=0,m6=600,m15=0,s12=false,k1=0,k16=0," + "s10=false,s21=false,k12=0,m5=600,s8=false,m4=600,m9=1107,s2=false,s13=false,s20=false,s23=false,k9=0,m11=1525," + "s4=false,s6=false 1672818929178749400", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k2=0,k11=0,m3=600,m12=1506,s17=false,m5=600,s11=false,s22=false,k6=0,m13=0,s16=true,k5=0,s21=false,m4=600,m7=" + "600,s9=false,s10=false,s18=false,k7=0,m8=600,k1=0,hex=" + "\"7b3a00000001030301030200000000323231313233304339344b30002b01012a10028003000000071105e8025802580258025802580258" + "044905eb05ef05e200000000000000000afc7d\",m11=1519,m16=0,s19=false,s23=false,s24=false,s14=false,s6=false,k10=0," + "k15=0,k14=0,s2=false,s4=false,s8=false,s13=false,s15=true,s20=false,m2=38.000005040000005,s3=false,s7=false,k3=" + "0,k8=0,k13=0,m6=600,m14=0,m15=0,k4=0,m1=20.450000000000003,m9=1097,s1=true,m10=1515,s5=false,s12=false,k9=0,k12=" + "0,k16=0 1672818919126971000", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "k7=0,k14=0,m3=600,m7=600,s5=false,k2=0,k3=0,k8=0,s3=false,s20=false,k15=0,m10=1482,s17=false,k1=0,k16=0,m15=0," + "s12=false,k9=0,m16=0,s11=false,m4=600,s10=false,s15=true,s24=false,m8=600,m13=0,s2=false,s18=false,k12=0,s14=" + "false,s19=false,hex=" + "\"7b3900000001030301030200000000323231313233304339344b30002b01012a10028003000000071505ef025802580258025802580258" + "045005ca05b105d800000000000000000aa47d\",s1=true,s4=false,s7=false,s8=false,s13=false,m6=600,s6=false,s21=false," + "k11=0,m12=1496,m9=1104,s16=true,k5=0,s9=false,k10=0,k13=0,m2=38.291671730000004,s22=false,m5=600,m11=1457,m14=0," + "k4=0,m1=20.650000000000006,s23=false,k6=0 1672818909130866800", + "DataRTU,deviceId=2211230C94K0_1,dataModelName=DataRTU_2211230C94K0_1 " + "m7=600,k4=0,k14=0,s22=false,k13=0,s2=false,m11=1510,m14=0,s4=false,s10=false,m1=21,m16=0,m13=0,s9=false,s13=" + "false,s14=false,k10=0,m3=600,m9=1107,s18=false,s19=false,k2=0,hex=" + "\"7b3600000001030301030200000000323231313233304339344b30002b01012a10028003000000071c0619025802580258025802580258" + "045305dc05e6058d00000000000000000ad27d\",m2=40.04167187,m8=600,k7=0,k8=0,m10=1500,s23=false,k5=0,s11=false,s21=" + "false,k9=0,m15=0,m12=1421,s1=true,s5=false,s8=false,m5=600,k16=0,k15=0,m6=600,s3=false,s6=false,s7=false,s15=" + "true,s20=false,s24=false,k11=0,k1=0,k6=0,k12=0,m4=600,s16=true,s17=false,k3=0,s12=false 1672818879189483200", + "DataRTU,deviceId=2106070C11M0_2,dataModelName=DataRTU_2106070C11M0_2 " + "m1=5691,k14=0,m6=0,s14=false,k8=0,s19=false,s20=false,k12=0,s17=false,k3=0,m8=0,s8=false,m7=0,s9=false,s4=false," + "s11=false,s13=false,s16=false,k5=0,k15=0,k16=0,s10=false,s23=false,s1=false,s2=false,s3=false,s12=false,s24=" + "false,k2=0,k10=0,hex=" + "\"7b1400000001030301030200000000323130363037304331314d30002b01022a080400000000000008af0c000000000000000000000000" + "000000000000000000000000000000000ad47d\",m2=0,s7=false,s18=false,s21=false,m3=0,m5=0,k4=0,k11=0,m4=0,k1=0,k6=0," + "k13=0,s6=false,s15=false,s5=false,s22=false,k7=0,k9=0 1672818779549848800"}; pRes = taos_query(taos, "use ts2385"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_NANO_SECONDS); + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); @@ -1168,14 +1069,14 @@ int sml_ts2385_Test() { pRes = taos_query(taos, "select distinct tbname from `DataRTU` order by tbname"); printf("%s result2:%s\n", __FUNCTION__, taos_errstr(pRes)); - int num = 0; + int num = 0; TAOS_ROW row = NULL; - while((row = taos_fetch_row(pRes))){ - if(row[0] != NULL && num == 0){ + while ((row = taos_fetch_row(pRes))) { + if (row[0] != NULL && num == 0) { ASSERT(strncmp((char *)row[0], "DataRTU_2106070C11M0_2", sizeof("DataRTU_2106070C11M0_2") - 1) == 0); } - if(row[0] != NULL && num == 1){ + if (row[0] != NULL && num == 1) { ASSERT(strncmp((char *)row[0], "DataRTU_2211230C94K0_1", sizeof("DataRTU_2211230C94K0_1") - 1) == 0); } num++; @@ -1190,13 +1091,17 @@ int sml_ts2385_Test() { } int main(int argc, char *argv[]) { - if(argc == 2){ + if (argc == 2) { taos_options(TSDB_OPTION_CONFIGDIR, argv[1]); } int ret = 0; - ret = sml_ts2385_Test(); + ret = sml_ts2385_Test(); // this test case need config sml table name using ./sml_test config_file ASSERT(!ret); + // for(int i = 0; i < sizeof(str)/sizeof(str[0]); i++){ + // printf("str:%s \t %d\n", str[i], smlCalTypeSum(str[i], strlen(str[i]))); + // } + // int ret = 0; ret = sml_ttl_Test(); ASSERT(!ret); ret = sml_ts2164_Test(); @@ -1211,8 +1116,6 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = smlProcess_json3_Test(); ASSERT(!ret); - ret = smlProcess_json4_Test(); - ASSERT(!ret); ret = sml_TD15662_Test(); ASSERT(!ret); ret = sml_TD15742_Test(); @@ -1221,12 +1124,8 @@ int main(int argc, char *argv[]) { ASSERT(!ret); ret = sml_oom_Test(); ASSERT(!ret); - ret = sml_16368_Test(); - ASSERT(!ret); ret = sml_dup_time_Test(); ASSERT(!ret); - ret = sml_16960_Test(); - ASSERT(!ret); ret = sml_add_tag_col_Test(); ASSERT(!ret); ret = smlProcess_18784_Test(); diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 024a253a2e2aad97b0639468b79ff04cb177b8a4..3d458d90c18edeb98ca7955346b0d20b9bca0923 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -21,20 +21,20 @@ #include "taos.h" #include "types.h" -static int running = 1; -TdFilePtr g_fp = NULL; -typedef struct{ +static int running = 1; +TdFilePtr g_fp = NULL; +typedef struct { bool snapShot; bool dropTable; bool subTable; int srcVgroups; int dstVgroups; char dir[64]; -}Config; +} Config; Config g_conf = {0}; -static TAOS* use_db(){ +static TAOS* use_db() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { return NULL; @@ -53,12 +53,12 @@ static void msg_process(TAOS_RES* msg) { printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); - TAOS *pConn = use_db(); + TAOS* pConn = use_db(); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { char* result = tmq_get_json_meta(msg); if (result) { printf("meta result: %s\n", result); - if(g_fp && strcmp(result, "") != 0){ + if (g_fp && strcmp(result, "") != 0) { taosFprintfFile(g_fp, result); taosFprintfFile(g_fp, "\n"); } @@ -71,35 +71,35 @@ static void msg_process(TAOS_RES* msg) { printf("write raw data type: %d\n", raw.raw_type); int32_t ret = tmq_write_raw(pConn, raw); printf("write raw data: %s\n", tmq_err2str(ret)); + ASSERT(ret == 0); + tmq_free_raw(raw); taos_close(pConn); } -int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ - +int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { /* test for TD-20612 start*/ -// pRes = taos_query(pConn,"create table tb1 (ts timestamp, c1 int, c2 int)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c1) values(1669092069069, 0)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn,"insert into tb1 (ts, c2) values(1669092069069, 1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); -// return -1; -// } -// taos_free_result(pRes); -// -// return 0; + pRes = taos_query(pConn, "create table tb1 (ts timestamp, c1 int, c2 int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tb1 (ts, c1) values(1669092069069, 0)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tb1 (ts, c2) values(1669092069069, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + /* test for TD-20612 end*/ pRes = taos_query(pConn, @@ -153,7 +153,10 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, 'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); + pRes = taos_query( + pConn, + "insert into ct3 values(1626006833600, 5, 6, 'c') ct1 values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " + "'ddd') ct0 values(1626006833603, 4, 3, 'hwj') ct1 values(now+5s, 23, 32, 's21ds')"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -174,7 +177,9 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) (1626006833609, 51, 62, 'c333', 940)"); + pRes = taos_query(pConn, + "insert into ct3 values(1626006833605, 53, 63, 'cffffffffffffffffffffffffffff', 8989898899999) " + "(1626006833609, 51, 62, 'c333', 940)"); if (taos_errno(pRes) != 0) { printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -209,8 +214,8 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - if(g_conf.dropTable){ - pRes = taos_query(pConn, "drop table ct3 ct1"); + if (g_conf.dropTable) { + pRes = taos_query(pConn, "drop table ct3, ct1"); if (taos_errno(pRes) != 0) { printf("failed to drop child table ct3, reason:%s\n", taos_errstr(pRes)); return -1; @@ -274,7 +279,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - if(g_conf.dropTable){ + if (g_conf.dropTable) { pRes = taos_query(pConn, "drop table n1"); if (taos_errno(pRes) != 0) { printf("failed to drop normal table n1, reason:%s\n", taos_errstr(pRes)); @@ -318,7 +323,7 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - if(g_conf.dropTable){ + if (g_conf.dropTable) { pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " "nchar(8), t4 bool)"); @@ -354,16 +359,22 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " - "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); + pRes = taos_query( + pConn, + "create table if not exists stt1 using stt tags(2, \"stt1\", true) sttb1 using sttb tags(4, \"sttb1\", true) " + "stt2 using stt tags(43, \"stt2\", false) sttb2 using sttb tags(54, \"sttb2\", true)"); if (taos_errno(pRes) != 0) { printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - pRes = taos_query(pConn, "insert into stt3 using stt tags(23, \"stt3\", true) values(now + 1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " - "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); + pRes = + taos_query(pConn, + "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " + "1s, 1, 2, 'stt3') sttb3 using sttb tags(4, \"sttb3\", true) values(now + 2s, 13, 22, 'sttb3') " + "stt4 using stt tags(433, \"stt4\", false) values(now + 3s, 21, 21, 'stt4') sttb4 using sttb " + "tags(543, \"sttb4\", true) values(now + 4s, 16, 25, 'sttb4')"); if (taos_errno(pRes) != 0) { printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -373,8 +384,10 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes){ return 0; } -int buildStable(TAOS* pConn, TAOS_RES* pRes){ - pRes = taos_query(pConn, "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` VARCHAR(16))"); +int buildStable(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "CREATE STABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS " + "(`groupid` INT, `location` VARCHAR(16))"); if (taos_errno(pRes) != 0) { printf("failed to create super table meters, reason:%s\n", taos_errstr(pRes)); return -1; @@ -395,7 +408,9 @@ int buildStable(TAOS* pConn, TAOS_RES* pRes){ } taos_free_result(pRes); - pRes = taos_query(pConn, "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, groupid, location from meters partition by groupid, location interval(10m)"); + pRes = taos_query(pConn, + "create stream meters_summary_s into meters_summary as select _wstart, max(current) as current, " + "groupid, location from meters partition by groupid, location interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create super table meters_summary, reason:%s\n", taos_errstr(pRes)); return -1; @@ -470,9 +485,9 @@ int32_t init_env() { } taos_free_result(pRes); - if(g_conf.subTable){ + if (g_conf.subTable) { buildStable(pConn, pRes); - }else{ + } else { buildDatabase(pConn, pRes); } @@ -495,14 +510,14 @@ int32_t create_topic() { } taos_free_result(pRes); - if(g_conf.subTable){ + if (g_conf.subTable) { pRes = taos_query(pConn, "create topic meters_summary_t1 with meta as stable meters_summary"); if (taos_errno(pRes) != 0) { printf("failed to create topic meters_summary_t1, reason:%s\n", taos_errstr(pRes)); return -1; } taos_free_result(pRes); - }else{ + } else { pRes = taos_query(pConn, "create topic topic_db with meta as database abc1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); @@ -529,7 +544,7 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.heartbeat.background", "true"); - if(g_conf.snapShot){ + if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); } @@ -542,9 +557,9 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); - if(g_conf.subTable){ + if (g_conf.subTable) { tmq_list_append(topic_list, "meters_summary_t1"); - }else{ + } else { tmq_list_append(topic_list, "topic_db"); } return topic_list; @@ -565,7 +580,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { cnt++; msg_process(tmqmessage); taos_free_result(tmqmessage); - }else{ + } else { break; } } @@ -581,10 +596,10 @@ void initLogFile() { char f1[256] = {0}; char f2[256] = {0}; - if(g_conf.snapShot){ + if (g_conf.snapShot) { sprintf(f1, "%s/../log/tmq_taosx_tmp_snapshot.source", g_conf.dir); sprintf(f2, "%s/../log/tmq_taosx_tmp_snapshot.result", g_conf.dir); - }else{ + } else { sprintf(f1, "%s/../log/tmq_taosx_tmp.source", g_conf.dir); sprintf(f2, "%s/../log/tmq_taosx_tmp.result", g_conf.dir); } @@ -602,82 +617,169 @@ void initLogFile() { exit(-1); } - if(g_conf.snapShot){ - if(g_conf.subTable){ - char *result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}" - }; - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + if (g_conf.snapShot) { + if (g_conf.subTable) { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" + "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" + "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" + "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" + "19}],\"createList\":[]}"}; + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { taosFprintfFile(pFile2, result[i]); taosFprintfFile(pFile2, "\n"); } - }else{ - char *result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}" - }; - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + } else { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":64},{" + "\"name\":\"c4\",\"type\":5}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":" + "8},{\"name\":\"t4\",\"type\":1},{\"name\":\"t2\",\"type\":8,\"length\":64}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":4,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":5000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c2\",\"type\":10,\"length\":8},{\"name\":\"cc3\",\"type\":5}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3," + "\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"}" + ",{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3," + "\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3," + "\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"}" + ",{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":0}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3," + "\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}"}; + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { taosFprintfFile(pFile2, result[i]); taosFprintfFile(pFile2, "\n"); } } - }else{ - if(g_conf.subTable){ - char *result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+19}],\"createList\":[]}" - }; - - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + } else { + if (g_conf.subTable) { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"meters_summary\",\"columns\":[{\"name\":\"_" + "wstart\",\"type\":9},{\"name\":\"current\",\"type\":6},{\"name\":\"groupid\",\"type\":4},{\"name\":" + "\"location\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"group_id\",\"type\":14}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"t_d2a450ee819dcf7576f0282d9ac22dbc\",\"using\":" + "\"meters_summary\",\"tagNum\":1,\"tags\":[{\"name\":\"group_id\",\"type\":14,\"value\":1.313555008277358e+" + "19}],\"createList\":[]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { taosFprintfFile(pFile2, result[i]); taosFprintfFile(pFile2, "\n"); } - }else{ - char *result[] = { - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\",\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\",\"colType\":8,\"colLength\":64}", - "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\",\"colValue\":\"5000\",\"colValueNull\":false}", + } else { + char* result[] = { + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"tb1\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":4}],\"tags\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"st1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct0\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":1000},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"ttt\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct1\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":2000}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct2\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"using\":\"st1\",\"tagNum\":3,\"tags\":[" + "{\"name\":\"t1\",\"type\":4,\"value\":3000}],\"createList\":[]}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":5,\"colName\":\"c4\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":7,\"colName\":\"c3\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"super\",\"tableName\":\"st1\",\"alterType\":1,\"colName\":\"t2\"," + "\"colType\":8,\"colLength\":64}", + "{\"type\":\"alter\",\"tableType\":\"child\",\"tableName\":\"ct3\",\"alterType\":4,\"colName\":\"t1\"," + "\"colValue\":\"5000\",\"colValueNull\":false}", "{\"type\":\"delete\",\"sql\":\"delete from `ct3` where `ts` >= 1626006833600 and `ts` <= 1626006833605\"}", - "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\",\"colType\":5}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\",\"colType\":10,\"colLength\":8}", - "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\",\"colNewName\":\"cc3\"}", + "{\"type\":\"create\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":10,\"length\":4}],\"tags\":[]}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":5,\"colName\":\"c3\"," + "\"colType\":5}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":7,\"colName\":\"c2\"," + "\"colType\":10,\"colLength\":8}", + "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":10,\"colName\":\"c3\"," + "\"colNewName\":\"cc3\"}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":9}", "{\"type\":\"alter\",\"tableType\":\"normal\",\"tableName\":\"n1\",\"alterType\":6,\"colName\":\"c1\"}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[],\"createList\":[]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}],\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":1}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", - "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\",\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}" - }; - - for(int i = 0; i < sizeof(result)/sizeof(result[0]); i++){ + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"jt\",\"columns\":[{\"name\":\"ts\",\"type\":9}," + "{\"name\":\"i\",\"type\":4}],\"tags\":[{\"name\":\"t\",\"type\":15}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt1\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[{" + "\"name\":\"t\",\"type\":15,\"value\":\"{\\\"k1\\\":1,\\\"k2\\\":\\\"hello\\\"}\"}],\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"jt2\",\"using\":\"jt\",\"tagNum\":1,\"tags\":[]" + ",\"createList\":[]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"stt\",\"columns\":[{\"name\":\"ts\",\"type\":9}" + ",{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"super\",\"tableName\":\"sttb\",\"columns\":[{\"name\":\"ts\",\"type\":" + "9},{\"name\":\"c1\",\"type\":4},{\"name\":\"c2\",\"type\":6},{\"name\":\"c3\",\"type\":8,\"length\":16}]," + "\"tags\":[{\"name\":\"t1\",\"type\":4},{\"name\":\"t3\",\"type\":10,\"length\":8},{\"name\":\"t4\",\"type\":" + "1}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt1\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt1\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt1\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":2},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb1\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb1\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt2\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":43},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb2\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":54},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb2\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}", + "{\"type\":\"create\",\"tableType\":\"child\",\"tableName\":\"stt3\",\"using\":\"stt\",\"tagNum\":3,\"tags\":" + "[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":\"\\\"stt3\\\"\"},{" + "\"name\":\"t4\",\"type\":1,\"value\":1}],\"createList\":[{\"tableName\":\"stt3\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":23},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"sttb3\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":4},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb3\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]},{\"tableName\":\"stt4\",\"using\":\"stt\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":433},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"stt4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":0}]},{\"tableName\":\"sttb4\",\"using\":\"sttb\"," + "\"tagNum\":3,\"tags\":[{\"name\":\"t1\",\"type\":4,\"value\":543},{\"name\":\"t3\",\"type\":10,\"value\":" + "\"\\\"sttb4\\\"\"},{\"name\":\"t4\",\"type\":1,\"value\":1}]}]}"}; + + for (int i = 0; i < sizeof(result) / sizeof(result[0]); i++) { taosFprintfFile(pFile2, result[i]); taosFprintfFile(pFile2, "\n"); } @@ -689,23 +791,23 @@ void initLogFile() { int main(int argc, char* argv[]) { for (int32_t i = 1; i < argc; i++) { - if(strcmp(argv[i], "-c") == 0){ + if (strcmp(argv[i], "-c") == 0) { tstrncpy(g_conf.dir, argv[++i], sizeof(g_conf.dir)); - }else if(strcmp(argv[i], "-s") == 0){ + } else if (strcmp(argv[i], "-s") == 0) { g_conf.snapShot = true; - }else if(strcmp(argv[i], "-d") == 0){ + } else if (strcmp(argv[i], "-d") == 0) { g_conf.dropTable = true; - }else if(strcmp(argv[i], "-sv") == 0){ + } else if (strcmp(argv[i], "-sv") == 0) { g_conf.srcVgroups = atol(argv[++i]); - }else if(strcmp(argv[i], "-dv") == 0){ + } else if (strcmp(argv[i], "-dv") == 0) { g_conf.dstVgroups = atol(argv[++i]); - }else if(strcmp(argv[i], "-t") == 0){ + } else if (strcmp(argv[i], "-t") == 0) { g_conf.subTable = true; } } printf("env init\n"); - if(strlen(g_conf.dir) != 0){ + if (strlen(g_conf.dir) != 0) { initLogFile(); } diff --git a/utils/tsim/src/simExe.c b/utils/tsim/src/simExe.c index 258c61155769be2d3aef314ee1602a4ba1b8af3d..d82c948cf7b774703faaa8e70ee7b49bde9f8d12 100644 --- a/utils/tsim/src/simExe.c +++ b/utils/tsim/src/simExe.c @@ -31,7 +31,7 @@ void simLogSql(char *sql, bool useSharp) { taosFprintfFile(pFile, "%s;\n", sql); } - taosFsyncFile(pFile); + UNUSED(taosFsyncFile(pFile)); } }